From 72444f3b87cdaa7cce2bfebe08a21f6862c953cb Mon Sep 17 00:00:00 2001
From: Seebs <seebs@molecula.com>
Date: Mon, 13 Sep 2021 16:23:57 -0500
Subject: [PATCH 01/66] add clusters to MutexCheck test, fix silly bug revealed
 by doing so

The merge lists behavior was flawed in that it would drop one item
from the list per merge, which means that, with high replication
and low number of distinct items, it could even produce an empty
list.

The actual "is there anything wrong" logic is fine, but the list of
clashing values set for a given record is not.

Unfortunately this also doubles the time the test takes, to
21 seconds on MacOS. OW.
---
 api.go      | 8 ++++----
 api_test.go | 8 +++++++-
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/api.go b/api.go
index fe5723365..8e7fa7b2b 100644
--- a/api.go
+++ b/api.go
@@ -2692,9 +2692,9 @@ func mergeIDLists(dst []uint64, src []uint64) []uint64 {
 		return dst[i] < dst[j]
 	})
 	// dedup.
-	n := 0
+	n := 1
 	prev := dst[0]
-	for i := 0; i < len(dst); i++ {
+	for i := 1; i < len(dst); i++ {
 		if dst[i] != prev {
 			dst[n] = dst[i]
 			n++
@@ -2712,9 +2712,9 @@ func mergeKeyLists(dst []string, src []string) []string {
 		return dst[i] < dst[j]
 	})
 	// dedup.
-	n := 0
+	n := 1
 	prev := dst[0]
-	for i := 0; i < len(dst); i++ {
+	for i := 1; i < len(dst); i++ {
 		if dst[i] != prev {
 			dst[n] = dst[i]
 			n++
diff --git a/api_test.go b/api_test.go
index e4826374f..50a59a03e 100644
--- a/api_test.go
+++ b/api_test.go
@@ -860,7 +860,13 @@ type mutexCheckField struct {
 }
 
 func TestAPI_MutexCheck(t *testing.T) {
-	c := test.MustRunCluster(t, 3)
+	c := test.MustNewCluster(t, 3)
+	for _, c := range c.Nodes {
+		c.Config.Cluster.ReplicaN = 2
+	}
+	if err := c.Start(); err != nil {
+		t.Fatalf("starting cluster: %v", err)
+	}
 	defer c.Close()
 
 	m0 := c.GetNode(0)

From 575854df8aa643945793eb4b80eaf00e539820c0 Mon Sep 17 00:00:00 2001
From: nagamocha3000 <nagamocha@gmail.com>
Date: Wed, 18 Aug 2021 17:34:00 +0300
Subject: [PATCH 02/66] Make index-key replication more resilient to network
 failures

---
 Makefile                                      |   7 +
 cluster.go                                    |   5 +
 holder.go                                     | 149 ++++++-
 internal/clustertests/cluster_test.go         |   2 +-
 .../docker-compose-index-key-replication.yml  |  73 ++++
 .../index_key_replication_test.go             | 375 ++++++++++++++++++
 server.go                                     |  13 +-
 7 files changed, 598 insertions(+), 26 deletions(-)
 create mode 100644 internal/clustertests/docker-compose-index-key-replication.yml
 create mode 100644 internal/clustertests/index_key_replication_test.go

diff --git a/Makefile b/Makefile
index 6fb679f91..67912d2f3 100644
--- a/Makefile
+++ b/Makefile
@@ -137,6 +137,13 @@ clustertests: vendor
 	docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1
 
 
+DOCKER_COMPOSE_INDEX_KEY_REPLICATION=internal/clustertests/docker-compose-index-key-replication.yml
+# Check clustertests target for more info
+clustertests-index-key-replication: vendor
+	docker-compose -f $(DOCKER_COMPOSE_INDEX_KEY_REPLICATION) down
+	docker-compose -f $(DOCKER_COMPOSE_INDEX_KEY_REPLICATION) build client1
+	docker-compose -f $(DOCKER_COMPOSE_INDEX_KEY_REPLICATION) up --exit-code-from=client1
+
 # Like clustertests, but rebuilds all images.
 clustertests-build: vendor
 	docker-compose -f $(DOCKER_COMPOSE) down -v
diff --git a/cluster.go b/cluster.go
index e86883c1c..6419f3681 100644
--- a/cluster.go
+++ b/cluster.go
@@ -1165,6 +1165,9 @@ func (c *cluster) followResizeInstruction(ctx context.Context, instr *ResizeInst
 		}
 	}
 
+	// fire off translation sync
+	_ = c.translationSyncer.Reset()
+
 	return nil
 }
 
@@ -1179,6 +1182,8 @@ func (c *cluster) resizeAbort() error {
 	if c.resizeCancel != nil {
 		c.resizeCancel()
 	}
+	// fire off translation sync
+	_ = c.translationSyncer.Reset()
 	return nil
 }
 
diff --git a/holder.go b/holder.go
index 0ec881add..0daf8416d 100644
--- a/holder.go
+++ b/holder.go
@@ -1472,7 +1472,10 @@ type holderSyncer struct {
 	Cluster *cluster
 
 	// Translation sync handling.
-	readers []TranslateEntryReader
+	readers                     []TranslateEntryReader
+	readersMu                   sync.Mutex
+	pendingReaders              int
+	stopInitializeReplicationCh chan struct{}
 
 	syncers errgroup.Group
 
@@ -1596,6 +1599,23 @@ func (s *holderSyncer) syncFragment(index, field, view string, shard uint64) err
 
 // resetTranslationSync reinitializes streaming sync of translation data.
 func (s *holderSyncer) resetTranslationSync() error {
+	if s.stopInitializeReplicationCh == nil {
+		// suppose stopTranslationSync[S] holds the lock s.readersMu and tries
+		// to send a signal to s.stopInitializeRepliationCh. If
+		// s.stopInitializeReplicationCh is unbufferd and
+		// s.initializeReplication[I] is trying to acquire s.readersMu, this
+		// will result in a deadlock.
+		// Hence, the channel is buffered to prevent this scenario. Once
+		// [S] releases the lock, [I] will acquire it, however, the value
+		// of s.pendingReaders will be -1, at this point [I] should not
+		// attempt to add any more readers since those readers will be 'stale'
+		// [I] should also drain the channel to prevent the next invocation
+		// of [I] receiving a stop signal that was meant for the current one
+		// This is needlessly complicated and is the result of me running
+		// into various deadlocks while trying to fix handling of column
+		// key replication.
+		s.stopInitializeReplicationCh = make(chan struct{})
+	}
 	// Stop existing streams.
 	if err := s.stopTranslationSync(); err != nil {
 		return errors.Wrap(err, "stop translation sync")
@@ -1655,7 +1675,15 @@ func newActiveTranslationSyncer(ch chan struct{}) *activeTranslationSyncer {
 
 // Reset resets the server's translation syncer.
 func (a *activeTranslationSyncer) Reset() error {
-	a.ch <- struct{}{}
+	// just in case some other part of the code has fired
+	// off a translation sync and it hasn't been received yet
+	// therefore we don't want to block on send since a.ch is
+	// (for now) unbuffered. One translationSync is as good as
+	// another
+	select {
+	case a.ch <- struct{}{}:
+	default:
+	}
 	return nil
 }
 
@@ -1665,6 +1693,17 @@ func (a *activeTranslationSyncer) Reset() error {
 // to complete. This should be called before reconnecting to the cluster in case
 // of a cluster resize or schema change.
 func (s *holderSyncer) stopTranslationSync() error {
+	s.readersMu.Lock()
+	defer func() {
+		s.readers = nil // will be populated by initializeReplication
+		s.readersMu.Unlock()
+	}()
+	// send signal to stop initializing more readers
+	if s.pendingReaders > 0 {
+		s.pendingReaders = -1
+		close(s.stopInitializeReplicationCh)
+		s.stopInitializeReplicationCh = make(chan struct{})
+	}
 	var g errgroup.Group
 	for i := range s.readers {
 		rd := s.readers[i]
@@ -1724,7 +1763,6 @@ func (s *holderSyncer) setTranslateReadOnlyFlags(snap *topology.ClusterSnapshot)
 // replicate these from whichever node is primary for that partition).
 func (s *holderSyncer) initializeReplication(snap *topology.ClusterSnapshot) error {
 	nodeMaps := make(map[string]TranslateOffsetMap)
-
 	if snap.ReplicaN > 1 {
 		if err := s.populateIndexReplication(nodeMaps, snap); err != nil {
 			return err
@@ -1734,26 +1772,99 @@ func (s *holderSyncer) initializeReplication(snap *topology.ClusterSnapshot) err
 		return err
 	}
 
+	// filter out empty nodes
+	nodes := make(map[*topology.Node]bool)
 	for _, node := range snap.Nodes {
 		m := nodeMaps[node.ID]
-		if m.Empty() {
-			continue
+		if !m.Empty() {
+			nodes[node] = true
+		}
+	}
+
+	// connect to remote nodes and set up readers
+	readersCh := make(chan TranslateEntryReader, len(nodes))
+	ctx, cancelAddingMoreReaders := context.WithCancel(context.Background())
+	defer cancelAddingMoreReaders()
+	s.readersMu.Lock()
+	s.pendingReaders = len(nodes)
+	s.readersMu.Unlock()
+	go func() {
+		for {
+			for node := range nodes {
+				// check if ctx cancelled
+				// this means there was a signal sent to stop further init
+				// of readers
+				select {
+				case <-s.Closing:
+					return
+				case <-ctx.Done():
+					close(readersCh)
+					return
+				default:
+				}
+				// connect to remote node
+				m := nodeMaps[node.ID]
+				rd, err := s.Holder.OpenTranslateReader(context.Background(), node.URI.String(), m)
+				if err != nil {
+					continue
+				}
+				readersCh <- rd
+				delete(nodes, node)
+			}
+			if len(nodes) == 0 {
+				close(readersCh)
+				return
+			}
+			time.Sleep(10 * time.Second)
+
+		}
+	}()
+
+	for {
+		select {
+		case <-s.Closing:
+			cancelAddingMoreReaders()
+			return nil
+		case <-s.stopInitializeReplicationCh:
+			return nil
+		case rd, ok := <-readersCh:
+			// all translate readers have been launched, hence channel is
+			// closed
+			if !ok {
+				return nil
+			}
+			s.readersMu.Lock()
+			// [S] has been initiated and acquired the lock first
+			// at this point we should close the reader we've recieved rather
+			// than start replication on it.
+			// [S] should have already closed all the rest
+			// of the reads if they were still in action.
+			// we are also draining the channel since the signal for stopping
+			// further replication was meant for us
+			if s.pendingReaders == -1 {
+				rd.Close()
+				cancelAddingMoreReaders()
+			drain:
+				for {
+					select {
+					case <-s.stopInitializeReplicationCh:
+					default:
+						break drain
+					}
+				}
+				s.readersMu.Unlock()
+				return nil
+			}
+			s.pendingReaders--
+			s.readers = append(s.readers, rd)
+			s.syncers.Go(func() error {
+				defer rd.Close()
+				s.readBothTranslateReader(rd, snap)
+				return nil
+			})
+			s.readersMu.Unlock()
 		}
-
-		// Connect to remote node and begin streaming.
-		rd, err := s.Holder.OpenTranslateReader(context.Background(), node.URI.String(), m)
-		if err != nil {
-			return err
-		}
-		s.readers = append(s.readers, rd)
-
-		s.syncers.Go(func() error {
-			defer rd.Close()
-			s.readBothTranslateReader(rd, snap)
-			return nil
-		})
 	}
-	return nil
 }
 
 // populateFieldReplication populates a map from node IDs to TranslateOffsetMaps
diff --git a/internal/clustertests/cluster_test.go b/internal/clustertests/cluster_test.go
index 892e66383..faad15337 100644
--- a/internal/clustertests/cluster_test.go
+++ b/internal/clustertests/cluster_test.go
@@ -21,7 +21,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/molecula/featurebase/v2"
+	pilosa "github.com/molecula/featurebase/v2"
 	"github.com/molecula/featurebase/v2/disco"
 	picli "github.com/molecula/featurebase/v2/http"
 )
diff --git a/internal/clustertests/docker-compose-index-key-replication.yml b/internal/clustertests/docker-compose-index-key-replication.yml
new file mode 100644
index 000000000..6f84d5947
--- /dev/null
+++ b/internal/clustertests/docker-compose-index-key-replication.yml
@@ -0,0 +1,73 @@
+version: '2'
+services: 
+  pilosa1:
+    build:
+      context: ../..
+      dockerfile: Dockerfile-clustertests
+    image: ptest
+    ports:
+      - "33455:10101"
+    environment:
+      - PILOSA_NAME=pilosa1
+      - PILOSA_ETCD_LISTEN_CLIENT_ADDRESS=http://0.0.0.0:10201
+      - PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS=http://pilosa1:10201
+      - PILOSA_ETCD_LISTEN_PEER_ADDRESS=http://0.0.0.0:10301
+      - PILOSA_ETCD_ADVERTISE_PEER_ADDRESS=http://pilosa1:10301
+      - PILOSA_ETCD_INITIAL_CLUSTER=pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301,pilosa3=http://pilosa3:10301
+      - PILOSA_CLUSTER_REPLICAS=3
+    networks:
+      - pilosanet
+    command:
+      - "/featurebase server --bind pilosa1:10101"
+  pilosa2:
+    build:
+      context: ../..
+      dockerfile: Dockerfile-clustertests
+    image: ptest
+    ports:
+      - "33456:10101"
+    environment:
+      - PILOSA_NAME=pilosa2
+      - PILOSA_ETCD_LISTEN_CLIENT_ADDRESS=http://0.0.0.0:10201
+      - PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS=http://pilosa2:10201
+      - PILOSA_ETCD_LISTEN_PEER_ADDRESS=http://0.0.0.0:10301
+      - PILOSA_ETCD_ADVERTISE_PEER_ADDRESS=http://pilosa2:10301
+      - PILOSA_ETCD_INITIAL_CLUSTER=pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301,pilosa3=http://pilosa3:10301
+      - PILOSA_CLUSTER_REPLICAS=3
+    networks:
+      - pilosanet
+    command:
+      - "/featurebase server --bind pilosa2:10101"
+  pilosa3:
+    build:
+      context: ../..
+      dockerfile: Dockerfile-clustertests
+    image: ptest
+    ports:
+      - "33457:10101"
+    environment:
+      - PILOSA_NAME=pilosa3
+      - PILOSA_ETCD_LISTEN_CLIENT_ADDRESS=http://0.0.0.0:10201
+      - PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS=http://pilosa3:10201
+      - PILOSA_ETCD_LISTEN_PEER_ADDRESS=http://0.0.0.0:10301
+      - PILOSA_ETCD_ADVERTISE_PEER_ADDRESS=http://pilosa3:10301
+      - PILOSA_ETCD_INITIAL_CLUSTER=pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301,pilosa3=http://pilosa3:10301
+      - PILOSA_CLUSTER_REPLICAS=3
+    networks:
+      - pilosanet
+    command:
+      - "/featurebase server --bind pilosa3:10101"
+  client1:
+    build:
+      context: .
+    environment:
+      - ENABLE_PILOSA_CLUSTER_TESTS_FOR_INDEX_KEY_REPLICATION=1
+      - GO111MODULE=on
+    networks:
+      - pilosanet
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    command:
+      - "cd /go/src/github.com/molecula/featurebase/ && go test -mod=vendor -v -run=IndexKey -count=1 github.com/molecula/featurebase/v2/internal/clustertests"
+networks: 
+  pilosanet:
diff --git a/internal/clustertests/index_key_replication_test.go b/internal/clustertests/index_key_replication_test.go
new file mode 100644
index 000000000..54c3dfa63
--- /dev/null
+++ b/internal/clustertests/index_key_replication_test.go
@@ -0,0 +1,375 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package clustertest
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"math/rand"
+	"os"
+	"os/exec"
+	"sync"
+	"testing"
+	"time"
+
+	pilosa "github.com/molecula/featurebase/v2"
+	"github.com/molecula/featurebase/v2/disco"
+	"github.com/molecula/featurebase/v2/http"
+	picli "github.com/molecula/featurebase/v2/http"
+	"github.com/molecula/featurebase/v2/net"
+	"github.com/molecula/featurebase/v2/topology"
+)
+
+// index -> key -> ids from all replicas
+type translationRes map[string]map[string][]uint64
+
+func defaultTranslationResults(indexes []string) translationRes {
+	res := make(translationRes)
+	for _, index := range indexes {
+		res[index] = make(map[string][]uint64)
+	}
+	return res
+}
+
+func verify(allRes translationRes, indexes []string, replicasN, count int) error {
+	received := 0
+	allIDsSame := func(ids []uint64) bool {
+		if len(ids) <= 1 {
+			// trivially true
+			return true
+		}
+		id := ids[0]
+		for _, other := range ids {
+			if id != other {
+				return false
+			}
+		}
+		return true
+	}
+	for _, index := range indexes {
+		res, ok := allRes[index]
+		if !ok {
+			return fmt.Errorf("Expected index '%s' but not present", index)
+		}
+		for key, ids := range res {
+			// first id is after successful translation, rest are from
+			// translate readers
+			replicationCount := len(ids) - 1
+			received += replicationCount
+			if replicationCount != replicasN {
+				return fmt.Errorf("Count for ids for key '%s'(%d), index '%s' not equal than replicasN(%d)", key, replicationCount, index, replicasN)
+			}
+			if !allIDsSame(ids) {
+				return fmt.Errorf("Expected all ids for key '%s', index '%s' to be the same across the cluster %+v", key, index, ids)
+			}
+		}
+	}
+
+	if received != count {
+		return fmt.Errorf("Expected %d count of keys, received %d", count, received)
+	}
+
+	return nil
+}
+func getURIsFromAddresses(addrs []string) ([]*net.URI, error) {
+	uris := make([]*net.URI, 0, len(addrs))
+	for _, addr := range addrs {
+		uri, err := net.NewURIFromAddress(addr)
+		if err != nil {
+			return nil, err
+		}
+		uris = append(uris, uri)
+	}
+	return uris, nil
+}
+
+func getClients(addrs []string) ([]*http.InternalClient, error) {
+	clients := make([]*http.InternalClient, 0, len(addrs))
+	for _, addr := range addrs {
+		c, err := picli.NewInternalClient(addr, picli.GetHTTPClient(nil))
+		if err != nil {
+			return nil, err
+		}
+		clients = append(clients, c)
+	}
+	return clients, nil
+}
+
+func genIndexNames(indexCount int) []string {
+	indexNames := make([]string, 0, indexCount)
+	for i := 1; i <= indexCount; i++ {
+		indexNames = append(indexNames, fmt.Sprintf("idx-%d", i))
+	}
+	return indexNames
+}
+
+func parseDuration(t *testing.T, s string) time.Duration {
+	parsed, err := time.ParseDuration(s)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return parsed
+}
+
+func durationToSeconds(d time.Duration) string {
+	s := int(d.Seconds())
+	return fmt.Sprintf("%ds", s)
+}
+
+func TestIndexKeyReplication(t *testing.T) {
+	if os.Getenv("ENABLE_PILOSA_CLUSTER_TESTS_FOR_INDEX_KEY_REPLICATION") != "1" {
+		t.Skip("pilosa cluster tests for index key replication are not enabled")
+	}
+	// configurations for test
+	replicasN := 3
+	indexCount := 4
+	intervalDurationArg := "100ms"
+	totalInsertionDurationArg := "10s"
+	coolOffDurationArg := "5s"
+	numKeysToInsertPerDuration := 100
+	addresses := []string{"pilosa1:10101", "pilosa2:10101", "pilosa3:10101"}
+
+	intervalDuration := parseDuration(t, intervalDurationArg)
+	totalInsertionDuration := parseDuration(t, totalInsertionDurationArg)
+	coolOffDuration := parseDuration(t, coolOffDurationArg)
+
+	indexes := genIndexNames(indexCount)
+	clients, err := getClients(addresses)
+	cli := clients[0]
+	if err != nil {
+		t.Fatalf("on init clients from addresses: %v, %v", addresses, err)
+	}
+	uris, err := getURIsFromAddresses(addresses)
+	if err != nil {
+		t.Fatalf("on init clients from addresses: %v, %v", addresses, err)
+	}
+	ctx := context.Background()
+
+	// create index keyed
+	for _, index := range indexes {
+		err = cli.EnsureIndex(ctx, index, pilosa.IndexOptions{
+			Keys: true,
+		})
+		if err != nil {
+			t.Fatalf("creating/asserting index: %v", err)
+		}
+	}
+
+	// set up index keys to insert
+	keysInserted := 0
+	allRes := defaultTranslationResults(indexes)
+	{
+		ctxForIndexKeyCreation, cancelFurtherInsertions := context.WithCancel(ctx)
+		var wg sync.WaitGroup
+		var mu = &sync.Mutex{}
+		wg.Add(len(indexes))
+		seed := time.Now().UnixNano()
+		t.Logf("start inserting index keys for %v, seed(%v)", totalInsertionDuration, seed)
+		rng := rand.New(rand.NewSource(seed))
+		for _, index := range indexes {
+			translations := allRes[index]
+			go func(index string, translations map[string][]uint64) {
+				keys := make([]string, numKeysToInsertPerDuration)
+				offset := 1
+				ticker := time.NewTicker(intervalDuration)
+				defer func() {
+					ticker.Stop()
+					wg.Done()
+				}()
+				for {
+					select {
+					case <-ticker.C:
+						for i := 0; i < numKeysToInsertPerDuration; i++ {
+							keys[i] = fmt.Sprintf("key-%d", i+offset)
+						}
+						// pick random node to send key creation to
+						r := rng.Intn(len(addresses))
+						cli, uri := clients[r], uris[r]
+
+						// insert index keys
+						transmap, err := cli.CreateIndexKeysNode(ctxForIndexKeyCreation, uri, index, keys...)
+						if err != nil {
+							if err == context.Canceled {
+								return
+							} else {
+								t.Logf("creating index keys for index(%s) send to node(%s): %v", index, uri.String(), err)
+								continue
+							}
+						}
+						for key, id := range transmap {
+							translations[key] = append(translations[key], id)
+						}
+						mu.Lock()
+						keysInserted += len(transmap)
+						mu.Unlock()
+						offset += numKeysToInsertPerDuration
+					case <-ctxForIndexKeyCreation.Done():
+						return
+					}
+				}
+			}(index, translations)
+		}
+		// inject fault
+		pcmd := exec.Command("/pumba", "netem", "--duration",
+			durationToSeconds(totalInsertionDuration),
+			"loss", "--percent", "50", "--correlation", "60",
+			"clustertests_pilosa3_1")
+		pcmd.Stdout = os.Stdout
+		pcmd.Stderr = os.Stderr
+		t.Logf("sending pumba fault injection cmd: %v", pcmd.String())
+		err = pcmd.Start()
+		if err != nil {
+			t.Fatalf("starting pumba command: %v", err)
+		}
+		err = pcmd.Wait()
+		if err != nil {
+			t.Fatalf("waiting on pumba pause cmd: %v", err)
+		}
+
+		// wait for index keys to be created
+		t.Logf("start wait to complete index key creation")
+		time.Sleep(totalInsertionDuration)
+		cancelFurtherInsertions()
+		wg.Wait()
+		t.Logf("done with inserting index keys. Total keys inserted: %d", keysInserted)
+	}
+
+	t.Logf("start cool off period: %v\n", coolOffDuration)
+	time.Sleep(coolOffDuration)
+	t.Log("done with cool off period, waiting for stability")
+	waitForStatus(t, clients[0].Status, string(disco.ClusterStateNormal), 30, 1*time.Second)
+	t.Log("done with waiting for stability, starting verifying persistence of index keys")
+
+	// get all nodes
+	nodes, err := cli.Nodes(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// prepare translate offset maps
+	nodeMaps := make(map[string]pilosa.TranslateOffsetMap)
+	for _, n := range nodes {
+		nodeMaps[n.ID] = make(pilosa.TranslateOffsetMap)
+	}
+	schema, err := cli.Schema(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, indexInfo := range schema {
+		index := indexInfo.Name
+		isKeyed := indexInfo.Options.Keys
+		if !isKeyed {
+			continue
+		}
+
+		partitionN := topology.DefaultPartitionN
+		for partition := 0; partition < partitionN; partition++ {
+			nodes, err := cli.PartitionNodes(ctx, partition)
+			if err != nil {
+				t.Fatal(err)
+			}
+			for _, n := range nodes {
+				m := nodeMaps[n.ID]
+				m.SetIndexPartitionOffset(index, partition, 0)
+			}
+		}
+	}
+
+	// open translate reader
+	readers := make([]pilosa.TranslateEntryReader, 0, len(nodes))
+	closeAllTranslateReaders := func() []error {
+		var errs []error
+		for _, tr := range readers {
+			err := tr.Close()
+			if err != nil {
+				errs = append(errs, err)
+			}
+		}
+		return errs
+	}
+	var tlsConfig *tls.Config = nil
+	var wg sync.WaitGroup
+	entries := make(chan *pilosa.TranslateEntry)
+	for _, n := range nodes {
+		client := http.GetHTTPClient(tlsConfig)
+		nodeURL := n.URI.String()
+		offsets := nodeMaps[n.ID]
+		openTranslateReader := http.GetOpenTranslateReaderFunc(client)
+		tr, err := openTranslateReader(ctx, nodeURL, offsets)
+		if err != nil {
+			t.Fatal(err)
+		}
+		wg.Add(1)
+		readers = append(readers, tr)
+		go func(node *topology.Node, tr pilosa.TranslateEntryReader) {
+			defer func() {
+				wg.Done()
+			}()
+			for {
+				var entry pilosa.TranslateEntry
+				err := tr.ReadEntry(&entry)
+				if err != nil {
+					// TODO also ignore transient http: read on clsoed response body errors
+					// for now just print error
+					if err != context.Canceled {
+						fmt.Printf("node(%s) On read from translate entry reader: %v", node.URI.String(), err)
+					}
+					return
+				}
+				// ignore field keys
+				if entry.Field != "" {
+					continue
+				}
+				entries <- &entry
+			}
+		}(n, tr)
+	}
+
+	// receive all translation entries made
+	count := replicasN * keysInserted
+	i := 0
+	for {
+		entry := <-entries
+		res, ok := allRes[entry.Index]
+		if !ok {
+			// ignore indexes we did not create for this test
+			continue
+		}
+		ids, ok := res[entry.Key]
+		if !ok {
+			// ignore keys we did not insert for the test
+			continue
+		}
+		res[entry.Key] = append(ids, entry.ID)
+		i++
+		if i == count {
+			break
+		}
+	}
+	errs := closeAllTranslateReaders()
+	wg.Wait()
+	for _, err := range errs {
+		if err != nil {
+			t.Errorf("on close translate readers: %v", err)
+		}
+	}
+	close(entries)
+
+	err = verify(allRes, indexes, replicasN, count)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
diff --git a/server.go b/server.go
index c9b4d8d5e..93d9947a0 100644
--- a/server.go
+++ b/server.go
@@ -437,7 +437,7 @@ func NewServer(opts ...ServerOption) (*Server, error) {
 		confirmDownRetries: defaultConfirmDownRetries,
 		confirmDownSleep:   defaultConfirmDownSleep,
 
-		resetTranslationSyncCh: make(chan struct{}),
+		resetTranslationSyncCh: make(chan struct{}, 1),
 
 		logger: logger.NopLogger,
 	}
@@ -569,11 +569,6 @@ func (s *Server) Open() error {
 		log.Println(errors.Wrap(err, "logging startup"))
 	}
 
-	// Start background process listening for translation
-	// sync resets.
-	s.wg.Add(1)
-	go func() { defer s.wg.Done(); s.monitorResetTranslationSync() }()
-
 	// Start DisCo.
 	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
 	defer cancel()
@@ -612,6 +607,12 @@ func (s *Server) Open() error {
 	s.syncer.Closing = s.closing
 	s.syncer.Stats = s.holder.Stats.WithTags("component:HolderSyncer")
 
+	// Start background process listening for translation
+	// sync resets.
+	s.wg.Add(1)
+	go func() { defer s.wg.Done(); s.monitorResetTranslationSync() }()
+	go func() { _ = s.translationSyncer.Reset() }()
+
 	// Open holder.
 	func() {
 		s.holder.startMsgsMu.Lock()

From ba10b3dfdef1cc567ceee57f3eee1af22e6c6817 Mon Sep 17 00:00:00 2001
From: kcrodgers24 <kcrodgers@DESKTOP-5HMR5OG.localdomain>
Date: Thu, 9 Sep 2021 13:16:23 -0700
Subject: [PATCH 03/66] add support for ARM64

---
 Makefile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Makefile b/Makefile
index 67912d2f3..78652dad4 100644
--- a/Makefile
+++ b/Makefile
@@ -116,6 +116,7 @@ release: check-clean generate-statik-docker
 	$(MAKE) release-build GOOS=darwin GOARCH=amd64
 	$(MAKE) release-build GOOS=darwin GOARCH=arm64
 	$(MAKE) release-build GOOS=linux GOARCH=amd64
+	$(MAKE) release-build GOOS=linux GOARCH=arm64
 
 # Create release build tarballs for all supported platforms. Same as `release`, but without embedded Lattice UI.
 release-sans-ui: check-clean
@@ -123,6 +124,7 @@ release-sans-ui: check-clean
 	$(MAKE) release-build GOOS=darwin GOARCH=amd64
 	$(MAKE) release-build GOOS=darwin GOARCH=arm64
 	$(MAKE) release-build GOOS=linux GOARCH=amd64
+	$(MAKE) release-build GOOS=linux GOARCH=arm64
 
 # try (e.g.) internal/clustertests/docker-compose-replication2.yml
 DOCKER_COMPOSE=internal/clustertests/docker-compose.yml

From efcb0768db7debee65e334eb31cf29dc6609144f Mon Sep 17 00:00:00 2001
From: kcrodgers24 <kcrodgers24@DESKTOP-5HMR5OG.localdomain>
Date: Thu, 9 Sep 2021 15:36:09 -0700
Subject: [PATCH 04/66] wip

---
 Makefile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Makefile b/Makefile
index 78652dad4..99159f068 100644
--- a/Makefile
+++ b/Makefile
@@ -206,6 +206,7 @@ generate: generate-protoc generate-statik generate-stringer generate-pql
 # Create release using Docker
 docker-release:
 	$(MAKE) docker-build GOOS=linux GOARCH=amd64
+	$(MAKE) docker-build GOOS-linux GOARCH=arm64
 	$(MAKE) docker-build GOOS=darwin GOARCH=amd64
 	$(MAKE) docker-build GOOS=darwin GOARCH=arm64
 

From 2ae1de8ad652f8da9993c0f403ae27b46857408f Mon Sep 17 00:00:00 2001
From: "Kasey C. Rodgers" <49999391+kcrodgers24@users.noreply.github.com>
Date: Mon, 13 Sep 2021 13:03:06 -0700
Subject: [PATCH 05/66] Update Makefile

corrected typo in docker-release section
---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 99159f068..3db1059cc 100644
--- a/Makefile
+++ b/Makefile
@@ -206,7 +206,7 @@ generate: generate-protoc generate-statik generate-stringer generate-pql
 # Create release using Docker
 docker-release:
 	$(MAKE) docker-build GOOS=linux GOARCH=amd64
-	$(MAKE) docker-build GOOS-linux GOARCH=arm64
+	$(MAKE) docker-build GOOS=linux GOARCH=arm64
 	$(MAKE) docker-build GOOS=darwin GOARCH=amd64
 	$(MAKE) docker-build GOOS=darwin GOARCH=arm64
 

From f549dae625c31f6647357e0dad0900c81e2abb34 Mon Sep 17 00:00:00 2001
From: rachithrr <rachith.ramaswamy@molecula.com>
Date: Mon, 13 Sep 2021 14:05:53 -0500
Subject: [PATCH 06/66] CORE-777: Added DecimalAgg field in GroupCount to
 output decimal sum -created groupCountDecimal -added test

---
 executor.go      | 67 ++++++++++++++++++++++++++++++++++++------------
 executor_test.go | 40 ++++++++++++++++++++++++++++-
 2 files changed, 89 insertions(+), 18 deletions(-)

diff --git a/executor.go b/executor.go
index 69d96928f..a46ca6d2a 100644
--- a/executor.go
+++ b/executor.go
@@ -1825,10 +1825,14 @@ func (e *executor) executeSumCountShard(ctx context.Context, qcx *Qcx, index str
 	if err != nil {
 		return ValCount{}, errors.Wrap(err, "computing sum")
 	}
-	return ValCount{
+	out := ValCount{
 		Val:   int64(vsum) + (int64(vcount) * bsig.Base),
 		Count: int64(vcount),
-	}, nil
+	}
+	if field.Type() == FieldTypeDecimal {
+		out.FloatVal = float64(int64(vsum)+(int64(vcount)*bsig.Base)) / math.Pow(10, float64(bsig.Scale))
+	}
+	return out, nil
 }
 
 // executeMinShard calculates the min for bsiGroups on a shard.
@@ -3044,6 +3048,13 @@ func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c
 			aggType = "aggregate"
 		}
 	}
+	for _, res := range results {
+		if res.DecimalAgg != 0 && aggType == "sum" {
+			aggType = "decimalSum"
+			break
+		}
+	}
+
 	return NewGroupCounts(aggType, results...), nil
 }
 
@@ -3132,9 +3143,10 @@ func (fr FieldRow) String() string {
 type aggregateType int
 
 const (
-	nilAggregate      aggregateType = 0
-	sumAggregate      aggregateType = 1
-	distinctAggregate aggregateType = 2
+	nilAggregate        aggregateType = 0
+	sumAggregate        aggregateType = 1
+	distinctAggregate   aggregateType = 2
+	decimalSumAggregate aggregateType = 3
 )
 
 // GroupCounts is a list of GroupCount.
@@ -3152,6 +3164,8 @@ func (g *GroupCounts) AggregateColumn() string {
 		return "sum"
 	case distinctAggregate:
 		return "aggregate"
+	case decimalSumAggregate:
+		return "decimalSum"
 	default:
 		return ""
 	}
@@ -3176,6 +3190,8 @@ func NewGroupCounts(agg string, groups ...GroupCount) *GroupCounts {
 		aggType = sumAggregate
 	case "aggregate":
 		aggType = distinctAggregate
+	case "decimalSum":
+		aggType = decimalSumAggregate
 	case "":
 		aggType = nilAggregate
 	default:
@@ -3246,42 +3262,57 @@ func (g *GroupCounts) MarshalJSON() ([]byte, error) {
 	if len(groups) == 0 {
 		return []byte("[]"), nil
 	}
+
 	switch g.aggregateType {
 	case sumAggregate:
 		counts = *(*[]groupCountSum)(unsafe.Pointer(&groups))
 	case distinctAggregate:
 		counts = *(*[]groupCountAggregate)(unsafe.Pointer(&groups))
+	case decimalSumAggregate:
+		counts = *(*[]groupCountDecimalSum)(unsafe.Pointer(&groups))
 	}
 	return json.Marshal(counts)
 }
 
 // GroupCount represents a result item for a group by query.
 type GroupCount struct {
-	Group []FieldRow `json:"group"`
-	Count uint64     `json:"count"`
-	Agg   int64      `json:"-"`
+	Group      []FieldRow `json:"group"`
+	Count      uint64     `json:"count"`
+	Agg        int64      `json:"-"`
+	DecimalAgg float64    `json:"-"`
 }
 
 type groupCountSum struct {
-	Group []FieldRow `json:"group"`
-	Count uint64     `json:"count"`
-	Agg   int64      `json:"sum"`
+	Group      []FieldRow `json:"group"`
+	Count      uint64     `json:"count"`
+	Agg        int64      `json:"sum"`
+	DecimalAgg float64    `json:"-"`
 }
 
 type groupCountAggregate struct {
-	Group []FieldRow `json:"group"`
-	Count uint64     `json:"count"`
-	Agg   int64      `json:"aggregate"`
+	Group      []FieldRow `json:"group"`
+	Count      uint64     `json:"count"`
+	Agg        int64      `json:"aggregate"`
+	DecimalAgg float64    `json:"-"`
+}
+
+type groupCountDecimalSum struct {
+	Group      []FieldRow `json:"group"`
+	Count      uint64     `json:"count"`
+	Agg        int64      `json:"-"`
+	DecimalAgg float64    `json:"sum"`
 }
 
 var _ GroupCount = GroupCount(groupCountSum{})
 var _ GroupCount = GroupCount(groupCountAggregate{})
+var _ GroupCount = GroupCount(groupCountDecimalSum{})
 
 func (g *GroupCount) Clone() (r *GroupCount) {
 	r = &GroupCount{
-		Group: make([]FieldRow, len(g.Group)),
-		Count: g.Count,
-		Agg:   g.Agg,
+		Group:      make([]FieldRow, len(g.Group)),
+		Count:      g.Count,
+		Agg:        g.Agg,
+		DecimalAgg: g.DecimalAgg,
 	}
 	for i := range g.Group {
 		r.Group[i] = *(g.Group[i].Clone())
@@ -3306,6 +3337,7 @@ func mergeGroupCounts(a, b []GroupCount, limit int) []GroupCount {
 		case 0:
 			a[i].Count += b[j].Count
 			a[i].Agg += b[j].Agg
+			a[i].DecimalAgg += b[j].DecimalAgg
 			ret = append(ret, a[i])
 			i++
 			j++
@@ -7939,6 +7971,7 @@ func (gbi *groupByIterator) Next(ctx context.Context) (ret GroupCount, done bool
 				}
 				ret.Count = uint64(result.Count)
 				ret.Agg = result.Val
+				ret.DecimalAgg = result.FloatVal
 			}
 		}
 		if ret.Count == 0 {
diff --git a/executor_test.go b/executor_test.go
index 8440adb9d..414bf47b5 100644
--- a/executor_test.go
+++ b/executor_test.go
@@ -38,7 +38,7 @@ import (
 	"github.com/davecgh/go-spew/spew"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
-	"github.com/molecula/featurebase/v2"
+	pilosa "github.com/molecula/featurebase/v2"
 	"github.com/molecula/featurebase/v2/boltdb"
 	"github.com/molecula/featurebase/v2/disco"
 	"github.com/molecula/featurebase/v2/http"
@@ -5151,6 +5151,8 @@ func TestExecutor_GroupByStrings(t *testing.T) {
 	c.CreateField(t, "istring", pilosa.IndexOptions{Keys: true}, "v", pilosa.OptFieldTypeInt(0, 1000))
 	c.CreateField(t, "istring", pilosa.IndexOptions{Keys: true}, "vv", pilosa.OptFieldTypeInt(0, 1000))
 	c.CreateField(t, "istring", pilosa.IndexOptions{Keys: true}, "nv", pilosa.OptFieldTypeInt(-1000, 1000))
+	c.CreateField(t, "istring", pilosa.IndexOptions{Keys: true}, "dv", pilosa.OptFieldTypeDecimal(2))
+	c.CreateField(t, "istring", pilosa.IndexOptions{Keys: true}, "ndv", pilosa.OptFieldTypeDecimal(1))
 
 	if err := c.GetNode(0).API.Import(context.Background(), nil, &pilosa.ImportRequest{
 		Index:      "istring",
@@ -5167,6 +5169,8 @@ func TestExecutor_GroupByStrings(t *testing.T) {
 
 	var v1, v2, v3, v4, v5, v6, v7, v8, v9, v10 int64 = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10
 	var nv1, nv2, nv3, nv4 int64 = -1, -2, -3, -4
+	var dv1, dv2, dv3, dv4, dv5, dv6, dv7, dv8, dv9, dv10 int64 = 111, 222, 333, 444, 555, 666, 777, 888, 999, 1000
+	var ndv1, ndv2, ndv3, ndv4, ndv5, ndv6, ndv7, ndv8, ndv9, ndv10 int64 = -111, -222, -333, -444, -555, -666, -777, -888, -999, -1000
 	if err := m0.API.ImportValue(context.Background(), qcx, &pilosa.ImportValueRequest{
 		Index:      "istring",
 		Field:      "v",
@@ -5197,6 +5201,26 @@ func TestExecutor_GroupByStrings(t *testing.T) {
 		t.Fatalf("importing: %v", err)
 	}
 
+	if err := m0.API.ImportValue(context.Background(), qcx, &pilosa.ImportValueRequest{
+		Index:      "istring",
+		Field:      "dv",
+		Shard:      0,
+		ColumnKeys: []string{"c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8", "c9", "c10"},
+		Values:     []int64{dv1, dv2, dv3, dv4, dv5, dv6, dv7, dv8, dv9, dv10},
+	}); err != nil {
+		t.Fatalf("importing: %v", err)
+	}
+
+	if err := m0.API.ImportValue(context.Background(), qcx, &pilosa.ImportValueRequest{
+		Index:      "istring",
+		Field:      "ndv",
+		Shard:      0,
+		ColumnKeys: []string{"c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8", "c9", "c10"},
+		Values:     []int64{ndv1, ndv2, ndv3, ndv4, ndv5, ndv6, ndv7, ndv8, ndv9, ndv10},
+	}); err != nil {
+		t.Fatalf("importing: %v", err)
+	}
+
 	tests := []struct {
 		query    string
 		expected []pilosa.GroupCount
@@ -5221,6 +5245,20 @@ func TestExecutor_GroupByStrings(t *testing.T) {
 				{Group: []pilosa.FieldRow{{Field: "generals", RowID: 2, RowKey: "r2"}}, Count: 5, Agg: 30},
 			},
 		},
+		{
+			query: "GroupBy(Rows(generals), aggregate=Sum(field=dv))",
+			expected: []pilosa.GroupCount{
+				{Group: []pilosa.FieldRow{{Field: "generals", RowID: 1, RowKey: "r1"}}, Count: 5, Agg: 2775, DecimalAgg: 27.75},
+				{Group: []pilosa.FieldRow{{Field: "generals", RowID: 2, RowKey: "r2"}}, Count: 5, Agg: 3220, DecimalAgg: 32.20},
+			},
+		},
+		{
+			query: "GroupBy(Rows(generals), aggregate=Sum(field=ndv))",
+			expected: []pilosa.GroupCount{
+				{Group: []pilosa.FieldRow{{Field: "generals", RowID: 1, RowKey: "r1"}}, Count: 5, Agg: -2775, DecimalAgg: -277.5},
+				{Group: []pilosa.FieldRow{{Field: "generals", RowID: 2, RowKey: "r2"}}, Count: 5, Agg: -3220, DecimalAgg: -322.0},
+			},
+		},
 		{
 			query: "GroupBy(Rows(generals), aggregate=Sum(field=v), having=Condition(sum>25))",
 			expected: []pilosa.GroupCount{

From 3ae12391c79d0a3803876ab1ecfdb4e680ff0146 Mon Sep 17 00:00:00 2001
From: Seebs <seebs@molecula.com>
Date: Tue, 14 Sep 2021 15:48:19 -0500
Subject: [PATCH 07/66] callback logic fixes for intersectionCallback and
 containerCallback

The inner loop of intersectionCallbackArrayArray's "fast"
case has
	for len(ca) > 0 && ca[0] < va {
	}
so we do not leave that loop unless len(ca) is 0, or
ca[0] >= va.

We then return from the whole function if len(ca) is 0,
so the only way we finish one iteration of the outer for
loop is if ca[0] >= va. Thus, this can be an `if` rather
than a `for`.

We also fix the logic for ArrayRun to make it require fewer
tests and be clearer about why the tests work and clearer about
always making progress.

And, finally, the bitmap/range callback logic, and the underlying
"callback per bit in word" logic, were both badly broken. In
particular, if a range started and ended in the same word, it would
hit the values in that word twice, once with them incorrectly
shifted, but then it would further garble any offsets past the first
in a word. Eww.
---
 roaring/roaring.go               | 51 +++++++++++++++++++++++---------
 roaring/roaring_internal_test.go | 12 ++++++++
 2 files changed, 49 insertions(+), 14 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 9567b37fb..26504f97b 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -2997,6 +2997,7 @@ func ArrayCountRange(array []uint16, start, end int32) (n int32) {
 	return n
 }
 
+// BitmapCountRange counts bits set in [start,end).
 func BitmapCountRange(bitmap []uint64, start, end int32) int32 {
 	if roaringParanoia {
 		if start > end {
@@ -3033,15 +3034,16 @@ func BitmapCountRange(bitmap []uint64, start, end int32) int32 {
 }
 
 func callbackBits(w uint64, base uint16, fn func(uint16)) {
-	bit := uint16(0)
 	for w != 0 {
-		trail := bits.TrailingZeros64(w)
-		bit += uint16(trail)
-		w >>= (trail + 1)
-		fn(base + bit)
+		trail := uint16(bits.TrailingZeros64(w))
+		fn(base + trail)
+		base += trail + 1
+		w >>= trail + 1
 	}
 }
 
+// bitmapCallbackRange calls the provided function for every bit set in
+// bitmap in the range [start,end).
 func bitmapCallbackRange(bitmap []uint64, start, end int32, fn func(uint16)) {
 	if roaringParanoia {
 		if start > end {
@@ -3051,11 +3053,30 @@ func bitmapCallbackRange(bitmap []uint64, start, end int32, fn func(uint16)) {
 	i, j := start/64, end/64
 	// Special case when start and end fall in the same word.
 	if i == j {
-		offi, offj := uint(start%64), uint(64-end%64)
-		w := (bitmap[i] >> offi) << (offj + offi)
+		// So, we want to know the offsets. For instance, if start and end
+		// are 65 and 69, we might want i=1, offi=1, j=1, offj=5. Then we
+		// compute masks from offi (masking out 0x1, or (1<<offi)-1), and
+		// offj (masking out everything *but* (1<<offj)-1).
+		//
+		// But we can also use "x >> offi << offi" to trim the lowest offi
+		// bits, and "x << (64-offj) >> (64-offj)" to trim all but the
+		// lowest offj bits.
+		//
+		// We can then simplify slightly further: we use the inverted value
+		// as offj, and compute (w << offi) >> (offi + offj) << offi.
+		//
+		// But wait, you ask. What if offi+offj is too large! Well, then
+		// start and end were in the wrong order. We have 0 <= i <= j < 64.
+		// If x+i > 64, then x > (64-i). Thus, if (64-j)+i > 64, it
+		// follows that (64-j) > (64-i). So they'd have been in the wrong order.
+		// In which case, we correctly yield a value of (0 << offi), or 0,
+		// because nothing is between them.
+		offi, offj := uint(start%64), uint(64-(end%64))
+		w := (bitmap[i] << offj) >> (offi + offj) << offi
 		if w != 0 {
 			callbackBits(w, uint16(i)*64, fn)
 		}
+		return
 	}
 
 	// Count partial starting word.
@@ -4492,7 +4513,7 @@ func intersectionCallbackArrayArray(a, b *Container, fn func(uint16)) {
 	}
 	if (na << 2) < nb {
 		for _, va := range ca {
-			for cb[0] < va {
+			if cb[0] < va {
 				// try to skip ahead a bit faster
 				for len(cb) > 7 && cb[7] < va {
 					cb = cb[8:]
@@ -4530,13 +4551,15 @@ func intersectionCallbackArrayRun(a, b *Container, fn func(uint16)) {
 	na, nb := len(array), len(runs)
 	for i, j := 0, 0; i < na && j < nb; {
 		va, vb := array[i], runs[j]
-		if va < vb.Start {
-			i++
-		} else if va >= vb.Start && va <= vb.Last {
-			i++
-			fn(va)
-		} else if va > vb.Last {
+		if va > vb.Last {
 			j++
+			continue
+		}
+		// If we got here, va is either before or in the current run,
+		// so we're definitely done with this member of the array.
+		i++
+		if va >= vb.Start {
+			fn(va)
 		}
 	}
 }
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index ce6f61a7d..1aaa5fb88 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -4636,3 +4636,15 @@ func TestContainer_unionInPlace_ArrayUnionRun(t *testing.T) {
 		}
 	}
 }
+
+func TestIntersectionCallback(t *testing.T) {
+	var hits []uint16
+	cb := func(u uint16) {
+		hits = append(hits, u)
+	}
+	bm := []uint64{0, 5, 0}
+	bitmapCallbackRange(bm, 64, 69, cb)
+	if len(hits) != 2 || hits[0] != 64 || hits[1] != 66 {
+		t.Fatalf("expected 64, 66, got %d", hits)
+	}
+}

From e0dfde99340fd551cf6373c6c4925dab2c5ef6ed Mon Sep 17 00:00:00 2001
From: Seebs <seebs@molecula.com>
Date: Wed, 15 Sep 2021 12:36:39 -0500
Subject: [PATCH 08/66] appease gofmt

---
 roaring/roaring.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 26504f97b..cf32e7689 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -4621,7 +4621,7 @@ func intersectionCallbackArrayBitmap(a, b *Container, fn func(uint16)) {
 			break
 		}
 		off := val % 64
-		if (bitmap[i]>>off) & 1 != 0 {
+		if (bitmap[i]>>off)&1 != 0 {
 			fn(val)
 		}
 	}

From 12244dcbed689644c670de157305c17336b04051 Mon Sep 17 00:00:00 2001
From: Seebs <seebs@molecula.com>
Date: Wed, 15 Sep 2021 13:40:07 -0500
Subject: [PATCH 09/66] record stats for intersectionCallback under the right
 name

---
 roaring/roaring.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index cf32e7689..e35487a3e 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -4565,7 +4565,7 @@ func intersectionCallbackArrayRun(a, b *Container, fn func(uint16)) {
 }
 
 func intersectionCallbackRunRun(a, b *Container, fn func(uint16)) {
-	statsHit("intersectionCount/RunRun")
+	statsHit("intersectionCallback/RunRun")
 	ra, rb := a.runs(), b.runs()
 	na, nb := len(ra), len(rb)
 	for i, j := 0, 0; i < na && j < nb; {
@@ -4605,14 +4605,14 @@ func intersectionCallbackRunRun(a, b *Container, fn func(uint16)) {
 }
 
 func intersectionCallbackBitmapRun(a, b *Container, fn func(uint16)) {
-	statsHit("intersectionCount/BitmapRun")
+	statsHit("intersectionCallback/BitmapRun")
 	for _, iv := range b.runs() {
 		bitmapCallbackRange(a.bitmap(), int32(iv.Start), int32(iv.Last)+1, fn)
 	}
 }
 
 func intersectionCallbackArrayBitmap(a, b *Container, fn func(uint16)) {
-	statsHit("intersectionCount/ArrayBitmap")
+	statsHit("intersectionCallback/ArrayBitmap")
 	bitmap := b.bitmap()
 	ln := len(bitmap)
 	for _, val := range a.array() {
@@ -4628,7 +4628,7 @@ func intersectionCallbackArrayBitmap(a, b *Container, fn func(uint16)) {
 }
 
 func intersectionCallbackBitmapBitmap(a, b *Container, fn func(uint16)) {
-	statsHit("intersectionCount/BitmapBitmap")
+	statsHit("intersectionCallback/BitmapBitmap")
 	ab, bb := a.bitmap(), b.bitmap()
 	for i := range ab {
 		w := ab[i] & bb[i]

From e7e3331fb4be6c70a860f181e3632b9d9f3e754f Mon Sep 17 00:00:00 2001
From: Seebs <seebs@molecula.com>
Date: Wed, 15 Sep 2021 12:36:46 -0500
Subject: [PATCH 10/66] test intersectionCallback more carefully

This takes our reasonably broad selection of predefined container
types and tries intersectionCallback on each pair of them, comparing
results against the results of plain old intersect(). We've had
several intersectionCallback fixes recently; every one of them
produces test failures here if reverted or broken, so I have at
least some confidence in this coverage.

Similarly, test everything on containerCallback, verifying that
we get the same set of values called back that we get from Slice().

Both of these were verified with -coverprofile to actually be
hitting all the lines of code that aren't insane edge case
checks like "what if a run is in the wrong order".
---
 roaring/roaring_internal_test.go | 104 ++++++++++++++++++++++++++++---
 1 file changed, 96 insertions(+), 8 deletions(-)

diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 1aaa5fb88..c23adc6f4 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -4637,14 +4637,102 @@ func TestContainer_unionInPlace_ArrayUnionRun(t *testing.T) {
 	}
 }
 
-func TestIntersectionCallback(t *testing.T) {
-	var hits []uint16
-	cb := func(u uint16) {
-		hits = append(hits, u)
+func TestContainerCallback(t *testing.T) {
+	containers, err := InitContainerArchetypes()
+	if err != nil {
+		t.Fatalf("creating containers: %v", err)
 	}
-	bm := []uint64{0, 5, 0}
-	bitmapCallbackRange(bm, 64, 69, cb)
-	if len(hits) != 2 || hits[0] != 64 || hits[1] != 66 {
-		t.Fatalf("expected 64, 66, got %d", hits)
+	got := make([]uint16, 65536)
+	hit := func(u uint16) {
+		got = append(got, u)
+	}
+	var expected []uint16
+	// complain() wraps up some pretty-printing logic for this,
+	// but note also the closure trapping expected/got so we can
+	// just refer to them without passing them in.
+	complain := func(t *testing.T, msg string, args ...interface{}) {
+		l1 := len(expected)
+		l2 := len(got)
+		dotdot1 := ""
+		dotdot2 := ""
+		if l1 > 8 {
+			expected = expected[:8]
+			dotdot1 = "..."
+		}
+		if l2 > 8 {
+			got = got[:8]
+			dotdot2 = "..."
+		}
+		t.Fatalf("%s: expected %d%s, got %d%s", fmt.Sprintf(msg, args...), expected, dotdot1, got, dotdot2)
+	}
+	for t1, ci := range containers {
+		t.Run(ContainerArchetypeNames[t1], func(t *testing.T) {
+			for _, c1 := range ci {
+				got = got[:0]
+				expected = c1.Slice()
+				containerCallback(c1, hit)
+				if len(got) != len(expected) {
+					complain(t, "wrong length (%d vs %d)", len(expected), len(got))
+				}
+				for i := range got {
+					if got[i] != expected[i] {
+						complain(t, "element %d differs: expected %d, got %d", i, expected[i], got[i])
+					}
+				}
+			}
+		})
+	}
+}
+
+func TestIntersectionCallback(t *testing.T) {
+	containers, err := InitContainerArchetypes()
+	if err != nil {
+		t.Fatalf("creating containers: %v", err)
+	}
+	got := make([]uint16, 65536)
+	hit := func(u uint16) {
+		got = append(got, u)
+	}
+	var expected []uint16
+	// complain() wraps up some pretty-printing logic for this,
+	// but note also the closure trapping expected/got so we can
+	// just refer to them without passing them in.
+	complain := func(t *testing.T, msg string, args ...interface{}) {
+		l1 := len(expected)
+		l2 := len(got)
+		dotdot1 := ""
+		dotdot2 := ""
+		if l1 > 8 {
+			expected = expected[:8]
+			dotdot1 = "..."
+		}
+		if l2 > 8 {
+			got = got[:8]
+			dotdot2 = "..."
+		}
+		t.Fatalf("%s: expected %d%s, got %d%s", fmt.Sprintf(msg, args...), expected, dotdot1, got, dotdot2)
+	}
+	for t1, ci := range containers {
+		for t2, cj := range containers {
+			t.Run(fmt.Sprintf("%s-%s", ContainerArchetypeNames[t1], ContainerArchetypeNames[t2]), func(t *testing.T) {
+				for _, c1 := range ci {
+					for _, c2 := range cj {
+						got = got[:0]
+						expectedContainer := intersect(c1, c2)
+						expected = expectedContainer.Slice()
+						intersectionCallback(c1, c2, hit)
+						if len(got) != len(expected) {
+							complain(t, "wrong length (%d vs %d)", len(expected), len(got))
+						}
+						for i := range got {
+							if got[i] != expected[i] {
+								complain(t, "element %d differs: expected %d, got %d", i, expected[i], got[i])
+							}
+						}
+					}
+				}
+
+			})
+		}
 	}
 }

From 3e222d87711275c132f5ffd5618c2e5e7641b10a Mon Sep 17 00:00:00 2001
From: Matthew Jaffee <jaffee@pilosa.com>
Date: Tue, 14 Sep 2021 17:00:00 -0500
Subject: [PATCH 11/66] tweak to locking which should avoid stall/deadlock w/
 mutex check

The view.go change is straightforward and fairly obviously more
correct.

The field.go change avoids holding the field read lock for the
duration of the mutex check request. The thinking was that while the
read lock was held something else was attempting to get a write lock,
which blocked all other read locks and something was getting into a
loop. Seebs might have a more detailed explanation, but that's as far
as my understanding goes at the moment. I believe this change is safe
though as we don't read/modify any field level data structures after
grabbing the standard view.
---
 field.go | 8 +++++++-
 view.go  | 4 ++--
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/field.go b/field.go
index 793371272..a6c4b5e75 100644
--- a/field.go
+++ b/field.go
@@ -1098,9 +1098,15 @@ func (f *Field) MutexCheck(ctx context.Context, qcx *Qcx, details bool, limit in
 	if f.Type() != FieldTypeMutex {
 		return nil, errors.New("mutex check only valid for mutex fields")
 	}
+
+	// Rather than deferring the unlock, we grab the standard view
+	// from the field's viewMap and unlock immediately. This avoids
+	// holding the rlock for a potentially long time which blocks any
+	// write lock, and pending write locks block other read locks.
 	f.mu.RLock()
-	defer f.mu.RUnlock()
 	standard := f.viewMap[viewStandard]
+	f.mu.RUnlock()
+
 	if standard == nil {
 		// no standard view present means we've never needed to create it,
 		// so it has no bits set, so it has no extra bits set.
diff --git a/view.go b/view.go
index bb0eb3a91..ac94f64d8 100644
--- a/view.go
+++ b/view.go
@@ -300,8 +300,8 @@ func (v *view) Fragment(shard uint64) *fragment {
 
 // allFragments returns a list of all fragments in the view.
 func (v *view) allFragments() []*fragment {
-	v.mu.Lock()
-	defer v.mu.Unlock()
+	v.mu.RLock()
+	defer v.mu.RUnlock()
 
 	other := make([]*fragment, 0, len(v.fragments))
 	for _, fragment := range v.fragments {

From b4cbd45b84285b9563b8b306ff7ed84687f725ba Mon Sep 17 00:00:00 2001
From: Ben Johnson <benbjohnson@yahoo.com>
Date: Thu, 16 Sep 2021 15:21:25 -0600
Subject: [PATCH 12/66] Implement SQL GROUP BY

---
 planner.go      | 191 ++++++++++++++++++++++++++++++++++++++++++++----
 planner_test.go |  68 +++++++++++++++++
 2 files changed, 245 insertions(+), 14 deletions(-)

diff --git a/planner.go b/planner.go
index 7dd39096e..4b080d279 100644
--- a/planner.go
+++ b/planner.go
@@ -80,27 +80,87 @@ func (p *Planner) planAggregateSelectStatement(ctx context.Context, stmt *sql2.S
 		return nil, err
 	}
 
-	// TODO: Support multiple aggregate calls.
-	if len(stmt.Columns) > 1 {
-		return nil, fmt.Errorf("only one call allowed in aggregate query")
+	// Extract calls and grouped expressions from column list.
+	// TODO: Recursively traverse all expression trees.
+	var calls []*sql2.Call
+	var aliases []string
+	// var groupByCols []*sql2.Ident // TODO: Convert to QualifiedRef
+	for _, c := range stmt.Columns {
+		aliases = append(aliases, c.Name())
+
+		switch c := c.Expr.(type) {
+		case *sql2.Call:
+			calls = append(calls, c)
+		case *sql2.Ident:
+			// groupByCols = append(groupByCols, c)
+		default:
+			return nil, fmt.Errorf("unsupported expression type in aggregate query: %T", c)
+		}
 	}
 
-	// Extract aggregate call.
-	col := stmt.Columns[0]
-	var call *sql2.Call
-	switch expr := col.Expr.(type) {
-	case *sql2.Call:
-		call = expr
-	default:
-		return nil, fmt.Errorf("unsupported expression in aggregate query: %T", expr)
+	// TODO: Support multiple calls per query.
+	if len(calls) > 1 {
+		return nil, fmt.Errorf("only one aggregate call allowed")
 	}
 
-	callName := strings.ToUpper(sql2.IdentName(call.Name))
+	// Extract column names in GROUP BY clause.
+	var groupByColNames []string
+	for _, expr := range stmt.GroupByExprs {
+		switch expr := expr.(type) {
+		case *sql2.Ident:
+			groupByColNames = append(groupByColNames, expr.Name)
+		default:
+			return nil, fmt.Errorf("unsupported expression type in GROUP BY clause: %T", expr)
+		}
+	}
+
+	// Extract aggregate call and build execution node.
+	callName := strings.ToUpper(sql2.IdentName(calls[0].Name))
 	switch callName {
 	case "COUNT":
-		return NewCountNode(p.executor, indexName, col.Name(), cond), nil
+		if len(groupByColNames) == 0 {
+			return NewCountNode(p.executor, indexName, aliases[0], cond), nil
+		}
+
+		var aggregate *pql.Call
+		if calls[0].Distinct.IsValid() {
+			if len(calls[0].Args) != 1 {
+				return nil, fmt.Errorf("distinct count must have exactly one field specified")
+			}
+			field, ok := calls[0].Args[0].(*sql2.Ident)
+			if !ok {
+				return nil, fmt.Errorf("distinct count argument must be a field name")
+			}
+
+			aggregate = &pql.Call{
+				Name: "Count",
+				Children: []*pql.Call{{
+					Name: "Distinct",
+					Args: map[string]interface{}{"field": field.Name},
+				}},
+			}
+		}
+
+		return NewGroupByNode(p.executor, indexName, groupByColNames, aliases, aggregate, cond), nil
+
+	case "SUM":
+		if len(calls[0].Args) != 1 {
+			return nil, fmt.Errorf("sum must have exactly one field specified")
+		}
+		field, ok := calls[0].Args[0].(*sql2.Ident)
+		if !ok {
+			return nil, fmt.Errorf("sum argument must be a field name")
+		}
+
+		aggregate := &pql.Call{
+			Name: "Sum",
+			Args: map[string]interface{}{"field": field.Name},
+		}
+
+		return NewGroupByNode(p.executor, indexName, groupByColNames, aliases, aggregate, cond), nil
+
 	default:
-		return nil, fmt.Errorf("unsupported call in aggregate query: %T", callName)
+		return nil, fmt.Errorf("unsupported call in aggregate query: %s", callName)
 	}
 
 	// TODO: Support HAVING
@@ -699,3 +759,106 @@ func (n *CountNode) Next(ctx context.Context) error {
 }
 
 func (n *CountNode) Row() []interface{} { return n.row }
+
+// GroupByNode executes an aggregate with a GROUP BY against a FeatureBase index.
+type GroupByNode struct {
+	executor  *executor
+	indexName string
+	columns   []string
+	aliases   []string
+	aggregate *pql.Call
+	cond      *pql.Call
+
+	result *GroupCounts
+	index  int
+
+	row []interface{}
+}
+
+func NewGroupByNode(executor *executor, indexName string, columns, aliases []string, aggregate, cond *pql.Call) *GroupByNode {
+	return &GroupByNode{
+		executor:  executor,
+		indexName: indexName,
+		columns:   columns,
+		aliases:   aliases,
+		aggregate: aggregate,
+		cond:      cond,
+		row:       make([]interface{}, len(columns)+1),
+	}
+}
+
+func (n *GroupByNode) Columns() []string {
+	return append([]string{"_aggregate"}, n.columns...)
+}
+
+func (n *GroupByNode) First(ctx context.Context) error {
+	n.result = nil
+	return nil
+}
+
+func (n *GroupByNode) Next(ctx context.Context) (err error) {
+	// Fetch resultset if it doesn't exist yet.
+	if n.result == nil {
+		if n.result, err = n.fetch(ctx); err != nil {
+			return err
+		}
+	}
+
+	// Exit if no more rows exist.
+	if n.index >= len(n.result.groups) {
+		return sql.ErrNoRows
+	}
+
+	// Copy results into current row.
+	group := n.result.groups[n.index]
+	n.index++
+
+	if n.aggregate != nil {
+		n.row[0] = int64(group.Agg)
+	} else {
+		n.row[0] = int64(group.Count)
+	}
+
+	for i, g := range group.Group {
+		if g.Value != nil {
+			n.row[i+1] = *g.Value
+		} else if g.RowKey != "" {
+			n.row[i+1] = g.RowKey
+		} else {
+			n.row[i+1] = int64(g.RowID)
+		}
+	}
+
+	return nil
+}
+
+// fetch executes a call to compute the PQL results.
+func (n *GroupByNode) fetch(ctx context.Context) (*GroupCounts, error) {
+	call := &pql.Call{
+		Name: "GroupBy",
+		Args: map[string]interface{}{},
+	}
+
+	// Choose fields to group by.
+	for _, col := range n.columns {
+		call.Children = append(call.Children, &pql.Call{
+			Name: "Rows", Args: map[string]interface{}{"_field": col},
+		})
+	}
+
+	// Apply filter & aggregate, if set.
+	if n.aggregate != nil {
+		call.Args["aggregate"] = n.aggregate
+	}
+	if n.cond != nil {
+		call.Args["filter"] = n.cond
+	}
+
+	result, err := n.executor.Execute(ctx, n.indexName, &pql.Query{Calls: []*pql.Call{call}}, nil, nil)
+	if err != nil {
+		return nil, err
+	}
+	return result.Results[0].(*GroupCounts), nil
+}
+
+func (n *GroupByNode) Row() []interface{} { return n.row }
diff --git a/planner_test.go b/planner_test.go
index bb9ff3bd0..4748046b6 100644
--- a/planner_test.go
+++ b/planner_test.go
@@ -263,6 +263,74 @@ func TestPlanner_Select(t *testing.T) {
 	})
 }
 
+func TestPlanner_GroupBy(t *testing.T) {
+	c := test.MustRunCluster(t, 1)
+	defer c.Close()
+
+	i0, err := c.GetHolder(0).CreateIndex("i0", pilosa.IndexOptions{TrackExistence: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer i0.Close()
+
+	if _, err := i0.CreateField("x"); err != nil {
+		t.Fatal(err)
+	} else if _, err := i0.CreateField("y", pilosa.OptFieldTypeInt(0, 1000)); err != nil {
+		t.Fatal(err)
+	} else if _, err := i0.CreateField("z", pilosa.OptFieldTypeInt(0, 1000)); err != nil {
+		t.Fatal(err)
+	}
+
+	// Populate with data.
+	if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{
+		Index: "i0",
+		Query: `
+			Set(1, x=10)
+			Set(1, x=20)
+			Set(1, y=100)
+			Set(1, z=500)
+
+			Set(2, x=10)
+			Set(2, y=200)
+			Set(2, z=500)
+
+			Set(3, x=20)
+			Set(3, z=600)
+	`}); err != nil {
+		t.Fatal(err)
+	}
+
+	t.Run("Count", func(t *testing.T) {
+		results := mustQueryRows(t, c.GetNode(0).Server, `SELECT COUNT(*), x FROM i0 GROUP BY x`)
+		if diff := cmp.Diff([][]interface{}{
+			{int64(2), int64(10)},
+			{int64(2), int64(20)},
+		}, results); diff != "" {
+			t.Fatal(diff)
+		}
+	})
+
+	t.Run("DistinctCount", func(t *testing.T) {
+		results := mustQueryRows(t, c.GetNode(0).Server, `SELECT COUNT(DISTINCT z), x FROM i0 GROUP BY x`)
+		if diff := cmp.Diff([][]interface{}{
+			{int64(1), int64(10)},
+			{int64(2), int64(20)},
+		}, results); diff != "" {
+			t.Fatal(diff)
+		}
+	})
+
+	t.Run("Sum", func(t *testing.T) {
+		results := mustQueryRows(t, c.GetNode(0).Server, `SELECT sum(y), x FROM i0 GROUP BY x`)
+		if diff := cmp.Diff([][]interface{}{
+			{int64(300), int64(10)},
+			{int64(100), int64(20)},
+		}, results); diff != "" {
+			t.Fatal(diff)
+		}
+	})
+}
+
 func mustQueryRows(tb testing.TB, svr *pilosa.Server, q string) [][]interface{} {
 	tb.Helper()
 

From ac022ce0ab9a7004c1a2df25acbac0729dd01aa9 Mon Sep 17 00:00:00 2001
From: Ben Johnson <benbjohnson@yahoo.com>
Date: Tue, 21 Sep 2021 08:29:30 -0600
Subject: [PATCH 13/66] CORE-860: Handle SQL comments during scan

---
 sql2/scanner.go      | 12 ++++++++++++
 sql2/scanner_test.go |  3 +++
 2 files changed, 15 insertions(+)

diff --git a/sql2/scanner.go b/sql2/scanner.go
index 552034a83..105f321b1 100644
--- a/sql2/scanner.go
+++ b/sql2/scanner.go
@@ -104,6 +104,11 @@ func (s *Scanner) Scan() (pos Pos, token Token, lit string) {
 		case '+':
 			return pos, PLUS, "+"
 		case '-':
+			if s.peek() == '-' {
+				s.read()
+				s.skipComment()
+				continue
+			}
 			return pos, MINUS, "-"
 		case '*':
 			return pos, STAR, "*"
@@ -117,6 +122,13 @@ func (s *Scanner) Scan() (pos Pos, token Token, lit string) {
 	}
 }
 
+// skipComment reads all characters until the end of the line or EOF.
+func (s *Scanner) skipComment() {
+	for ch := s.peek(); ch != '\n' && ch != -1; ch = s.peek() {
+		s.read()
+	}
+}
+
 func (s *Scanner) scanUnquotedIdent(pos Pos, prefix string) (Pos, Token, string) {
 	assert(isUnquotedIdent(s.peek()))
 
diff --git a/sql2/scanner_test.go b/sql2/scanner_test.go
index 27e33222d..f4a25a0aa 100644
--- a/sql2/scanner_test.go
+++ b/sql2/scanner_test.go
@@ -38,6 +38,9 @@ func TestScanner_Scan(t *testing.T) {
 		t.Run("StartingX", func(t *testing.T) {
 			AssertScan(t, `xyz`, sql.IDENT, `xyz`)
 		})
+		t.Run("WithComment", func(t *testing.T) {
+			AssertScan(t, "-- this is a comment\n\n-- more comments\nfoo", sql.IDENT, `foo`)
+		})
 	})
 
 	t.Run("KEYWORD", func(t *testing.T) {

From 44897d93105e242173d99c9b5b19f9c540b7672a Mon Sep 17 00:00:00 2001
From: Samir Patel <48686912+54mir@users.noreply.github.com>
Date: Wed, 22 Sep 2021 15:35:52 -0500
Subject: [PATCH 14/66] docker fix

---
 Dockerfile-clustertests | 2 +-
 Makefile                | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Dockerfile-clustertests b/Dockerfile-clustertests
index 66fe174e9..91fde88fb 100644
--- a/Dockerfile-clustertests
+++ b/Dockerfile-clustertests
@@ -1,7 +1,7 @@
 # This Dockerfile is used for cluster testing - it produces a much larger image
 # and includes all of Go as well as some utilities.
 
-FROM golang:1.13
+FROM golang:1.16
 
 LABEL maintainer "dev@pilosa.com"
 
diff --git a/Makefile b/Makefile
index 3db1059cc..335685187 100644
--- a/Makefile
+++ b/Makefile
@@ -135,7 +135,7 @@ DOCKER_COMPOSE=internal/clustertests/docker-compose.yml
 # pilosa image.
 clustertests: vendor
 	docker-compose -f $(DOCKER_COMPOSE) down
-	docker-compose -f $(DOCKER_COMPOSE) build client1
+	docker-compose -f $(DOCKER_COMPOSE) build
 	docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1
 
 
@@ -143,7 +143,7 @@ DOCKER_COMPOSE_INDEX_KEY_REPLICATION=internal/clustertests/docker-compose-index-
 # Check clustertests target for more info
 clustertests-index-key-replication: vendor
 	docker-compose -f $(DOCKER_COMPOSE_INDEX_KEY_REPLICATION) down
-	docker-compose -f $(DOCKER_COMPOSE_INDEX_KEY_REPLICATION) build client1
+	docker-compose -f $(DOCKER_COMPOSE_INDEX_KEY_REPLICATION) build
 	docker-compose -f $(DOCKER_COMPOSE_INDEX_KEY_REPLICATION) up --exit-code-from=client1
 
 # Like clustertests, but rebuilds all images.

From 3ca15cc3c47b2db9de37cb62488b70ca04f0954e Mon Sep 17 00:00:00 2001
From: Samir Patel <48686912+54mir@users.noreply.github.com>
Date: Thu, 16 Sep 2021 14:52:06 -0500
Subject: [PATCH 15/66] Rename docker builds to featurebase

---
 Dockerfile           |  4 ++--
 Dockerfile.pilosa    |  4 ++--
 Dockerfile.runner    |  4 ++--
 Makefile             | 14 +++++++-------
 docker-compose-3.yml |  8 ++++----
 5 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 20170e420..db864ed81 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -28,7 +28,7 @@ COPY . ./
 COPY --from=lattice-builder /lattice/build /lattice
 RUN /go/bin/statik -src=/lattice -dest=/pilosa
 
-RUN make build FLAGS="-o build/pilosa" ${MAKE_FLAGS}
+RUN make build FLAGS="-o build/featurebase" ${MAKE_FLAGS}
 
 #####################
 ### Pilosa runner ###
@@ -40,7 +40,7 @@ LABEL maintainer "dev@molecula.com"
 
 RUN apk add --no-cache curl jq
 
-COPY --from=pilosa-builder /pilosa/build/pilosa /
+COPY --from=pilosa-builder /pilosa/build/featurebase /
 
 COPY LICENSE /LICENSE
 COPY NOTICE /NOTICE
diff --git a/Dockerfile.pilosa b/Dockerfile.pilosa
index 068279d14..c830a8862 100644
--- a/Dockerfile.pilosa
+++ b/Dockerfile.pilosa
@@ -10,7 +10,7 @@ WORKDIR /pilosa
 
 COPY . ./
 
-RUN make build FLAGS="-o build/pilosa" ${MAKE_FLAGS}
+RUN make build FLAGS="-o build/featurebase" ${MAKE_FLAGS}
 
 #####################
 ### Pilosa runner ###
@@ -22,7 +22,7 @@ LABEL maintainer "dev@molecula.com"
 
 RUN apk add --no-cache curl jq
 
-COPY --from=pilosa-builder /pilosa/build/pilosa /
+COPY --from=pilosa-builder /pilosa/build/featurebase /
 
 COPY LICENSE /LICENSE
 COPY NOTICE /NOTICE
diff --git a/Dockerfile.runner b/Dockerfile.runner
index 3b1be7fd9..12f49e13c 100644
--- a/Dockerfile.runner
+++ b/Dockerfile.runner
@@ -10,7 +10,7 @@ WORKDIR /pilosa
 
 COPY . ./
 
-RUN make build FLAGS="-o build/pilosa" ${MAKE_FLAGS}
+RUN make build FLAGS="-o build/featurebase" ${MAKE_FLAGS}
 
 FROM moleculacorp/idk as idk
 LABEL maintainer "dev@molecula.com"
@@ -18,6 +18,6 @@ RUN apt-get update -y
 RUN apt-get install -y bash curl jq
 
 
-COPY --from=pilosa-builder /pilosa/build/pilosa /
+COPY --from=pilosa-builder /pilosa/build/featurebase /
 COPY testBackupRestore.sh /
 CMD ["bash","/testBackupRestore.sh"]
diff --git a/Makefile b/Makefile
index 335685187..d56d9d284 100644
--- a/Makefile
+++ b/Makefile
@@ -216,13 +216,13 @@ docker-build: vendor
 	    --build-arg GO_VERSION=$(GO_VERSION) \
 	    --build-arg MAKE_FLAGS="TRIAL_DEADLINE=$(TRIAL_DEADLINE) GOOS=$(GOOS) GOARCH=$(GOARCH)" \
 	    --target pilosa-builder \
-	    --tag pilosa:build .
-	docker create --name pilosa-build pilosa:build
-	mkdir -p build/pilosa-$(VERSION_ID)
-	docker cp pilosa-build:/pilosa/build/. ./build/pilosa-$(VERSION_ID)
-	cp NOTICE LICENSE ./build/pilosa-$(VERSION_ID)
-	docker rm pilosa-build
-	tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/
+	    --tag featurebase:build .
+	docker create --name featurebase-build featurebase:build
+	mkdir -p build/featurebase-$(VERSION_ID)
+	docker cp featurebase-build:/pilosa/build/. ./build/featurebase-$(VERSION_ID)
+	cp NOTICE LICENSE ./build/featurebase-$(VERSION_ID)
+	docker rm featurebase-build
+	tar -cvz -C build -f build/featurebase-$(VERSION_ID).tar.gz featurebase-$(VERSION_ID)/
 
 # Create Docker image from Dockerfile
 docker-image: vendor
diff --git a/docker-compose-3.yml b/docker-compose-3.yml
index ef31424d0..1998fcf97 100644
--- a/docker-compose-3.yml
+++ b/docker-compose-3.yml
@@ -1,7 +1,7 @@
 version: "3"
 services:
   pilosa0:
-    image: build/pilosa                                                                                                                                                     
+    image: build/featurebase                                                                                                                                                     
     build:                                                                                                                                                                 
       context: .                                                                                                                                                           
       dockerfile: Dockerfile.pilosa              
@@ -25,7 +25,7 @@ services:
       timeout: 5s
       retries: 5
   pilosa1:
-    image: build/pilosa                                                                                                                                                     
+    image: build/featurebase                                                                                                                                                     
     build:                                                                                                                                                                 
       context: .                                                                                                                                                           
       dockerfile: Dockerfile.pilosa              
@@ -44,7 +44,7 @@ services:
     volumes:
     - data:/data
   pilosa2:
-    image: build/pilosa                                                                                                                                                     
+    image: build/featurebase                                                                                                                                                     
     build:                                                                                                                                                                 
       context: .                                                                                                                                                           
       dockerfile: Dockerfile.pilosa              
@@ -63,7 +63,7 @@ services:
     volumes:
     - data:/data
   pilosax:
-    image: build/pilosa                                                                                                                                                     
+    image: build/featurebase                                                                                                                                                     
     build:                                                                                                                                                                 
       context: .                                                                                                                                                           
       dockerfile: Dockerfile.pilosa              

From 0ec855766ca3e74d4babd07868ab1634a4f630c5 Mon Sep 17 00:00:00 2001
From: Samir Patel <48686912+54mir@users.noreply.github.com>
Date: Thu, 16 Sep 2021 16:38:40 -0500
Subject: [PATCH 16/66] Rename in make docker-image

---
 Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index d56d9d284..520c57e3b 100644
--- a/Makefile
+++ b/Makefile
@@ -229,8 +229,8 @@ docker-image: vendor
 	docker build \
 	    --build-arg GO_VERSION=$(GO_VERSION) \
 	    --build-arg MAKE_FLAGS="TRIAL_DEADLINE=$(TRIAL_DEADLINE)" \
-	    --tag pilosa:$(VERSION) .
-	@echo Created docker image: pilosa:$(VERSION)
+	    --tag featurebase:$(VERSION) .
+	@echo Created docker image: featurebase:$(VERSION)
 
 # Create docker image (alias)
 docker: docker-image # alias

From 746ffd4bd17841b899b08bbf9728b8d8c29f74f2 Mon Sep 17 00:00:00 2001
From: Samir Patel <48686912+54mir@users.noreply.github.com>
Date: Thu, 16 Sep 2021 16:42:11 -0500
Subject: [PATCH 17/66] Rename in docker-tag-push

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 520c57e3b..4a89aa66c 100644
--- a/Makefile
+++ b/Makefile
@@ -237,7 +237,7 @@ docker: docker-image # alias
 
 # Tag and push a Docker image
 docker-tag-push: vendor
-	docker tag "pilosa:$(VERSION)" $(DOCKER_TARGET)
+	docker tag "featurebase:$(VERSION)" $(DOCKER_TARGET)
 	docker push $(DOCKER_TARGET)
 	@echo Pushed docker image: $(DOCKER_TARGET)
 

From 4699c840bc8919ce7c4b5ed64c1be33f61e0b7f4 Mon Sep 17 00:00:00 2001
From: Samir Patel <48686912+54mir@users.noreply.github.com>
Date: Wed, 22 Sep 2021 11:50:31 -0500
Subject: [PATCH 18/66] More rename to work with backup and cluster tests

---
 Dockerfile           |  2 +-
 Dockerfile.pilosa    |  5 +++--
 docker-compose-3.yml |  8 ++++----
 testBackupRestore.sh | 12 ++++++------
 4 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index db864ed81..115855034 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -52,5 +52,5 @@ ENV PILOSA_DATA_DIR /data
 ENV PILOSA_BIND 0.0.0.0:10101
 ENV PILOSA_BIND_GRPC 0.0.0.0:20101
 
-ENTRYPOINT ["/pilosa"]
+ENTRYPOINT ["/featurebase"]
 CMD ["server"]
diff --git a/Dockerfile.pilosa b/Dockerfile.pilosa
index c830a8862..fd8629946 100644
--- a/Dockerfile.pilosa
+++ b/Dockerfile.pilosa
@@ -33,6 +33,7 @@ VOLUME /data
 ENV PILOSA_DATA_DIR /data
 ENV PILOSA_BIND 0.0.0.0:10101
 ENV PILOSA_BIND_GRPC 0.0.0.0:20101
-
-ENTRYPOINT ["/pilosa"]
+RUN echo RUNNING DOCKERFILE.PILOSA
+CMD echo DOCKERFILE.PILOSA
+ENTRYPOINT ["/featurebase"]
 CMD ["server"]
diff --git a/docker-compose-3.yml b/docker-compose-3.yml
index 1998fcf97..ef31424d0 100644
--- a/docker-compose-3.yml
+++ b/docker-compose-3.yml
@@ -1,7 +1,7 @@
 version: "3"
 services:
   pilosa0:
-    image: build/featurebase                                                                                                                                                     
+    image: build/pilosa                                                                                                                                                     
     build:                                                                                                                                                                 
       context: .                                                                                                                                                           
       dockerfile: Dockerfile.pilosa              
@@ -25,7 +25,7 @@ services:
       timeout: 5s
       retries: 5
   pilosa1:
-    image: build/featurebase                                                                                                                                                     
+    image: build/pilosa                                                                                                                                                     
     build:                                                                                                                                                                 
       context: .                                                                                                                                                           
       dockerfile: Dockerfile.pilosa              
@@ -44,7 +44,7 @@ services:
     volumes:
     - data:/data
   pilosa2:
-    image: build/featurebase                                                                                                                                                     
+    image: build/pilosa                                                                                                                                                     
     build:                                                                                                                                                                 
       context: .                                                                                                                                                           
       dockerfile: Dockerfile.pilosa              
@@ -63,7 +63,7 @@ services:
     volumes:
     - data:/data
   pilosax:
-    image: build/featurebase                                                                                                                                                     
+    image: build/pilosa                                                                                                                                                     
     build:                                                                                                                                                                 
       context: .                                                                                                                                                           
       dockerfile: Dockerfile.pilosa              
diff --git a/testBackupRestore.sh b/testBackupRestore.sh
index 8f9617e7f..6d0dbc67f 100755
--- a/testBackupRestore.sh
+++ b/testBackupRestore.sh
@@ -13,19 +13,19 @@ STATUS=$STATUS timeout -s TERM $TIMEOUT bash -c \
     done;'
 echo "NOW DO STUFF"
 datagen --source kitchensink_keyed -e 9999 --pilosa.index sink --pilosa.batch-size 10000 --pilosa.hosts pilosa0:10101
-before=$(/pilosa chksum --host pilosa0:10101)
-/pilosa backup -o backupdir --host pilosa0:10101
+before=$(/featurebase chksum --host pilosa0:10101)
+/featurebase backup -o backupdir --host pilosa0:10101
 curl -X DELETE -s pilosa0:10101/index/sink
-/pilosa restore -s backupdir --host pilosa0:10101
-after=$(/pilosa chksum --host pilosa0:10101)
+/featurebase restore -s backupdir --host pilosa0:10101
+after=$(/featurebase chksum --host pilosa0:10101)
 if [ "$before" = "$after" ]; then
 	echo "PASS Cluster"
 else
 	echo "FAIL Single"
 	exit 1
 fi
-/pilosa restore -s backupdir --host pilosax:10101
-single=$(/pilosa chksum --host pilosax:10101)
+/featurebase restore -s backupdir --host pilosax:10101
+single=$(/featurebase chksum --host pilosax:10101)
 if [ "$before" = "$single" ]; then
 	echo "PASS Single"
 	exit 0

From 839dd1cd43441130aa28d78ed83a30a842f595a7 Mon Sep 17 00:00:00 2001
From: Samir Patel <48686912+54mir@users.noreply.github.com>
Date: Wed, 22 Sep 2021 16:02:02 -0500
Subject: [PATCH 19/66] Remove prints

---
 Dockerfile.pilosa | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Dockerfile.pilosa b/Dockerfile.pilosa
index fd8629946..a24c70ec7 100644
--- a/Dockerfile.pilosa
+++ b/Dockerfile.pilosa
@@ -33,7 +33,6 @@ VOLUME /data
 ENV PILOSA_DATA_DIR /data
 ENV PILOSA_BIND 0.0.0.0:10101
 ENV PILOSA_BIND_GRPC 0.0.0.0:20101
-RUN echo RUNNING DOCKERFILE.PILOSA
-CMD echo DOCKERFILE.PILOSA
+
 ENTRYPOINT ["/featurebase"]
 CMD ["server"]

From 693606fa807c30ce4c0ce25e2fa852210533b19f Mon Sep 17 00:00:00 2001
From: Matthew Jaffee <jaffee@Matthews-MacBook-Pro.local>
Date: Tue, 10 Aug 2021 14:01:53 -0500
Subject: [PATCH 20/66] better explanation for 'cannot allocate memory' error

---
 syswrap/mmap.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/syswrap/mmap.go b/syswrap/mmap.go
index 238f6dd6f..2d89ce39b 100644
--- a/syswrap/mmap.go
+++ b/syswrap/mmap.go
@@ -20,6 +20,7 @@ import (
 	"sync"
 	"sync/atomic"
 	"syscall"
+	"strings"
 
 	"github.com/pkg/errors"
 )
@@ -56,6 +57,9 @@ func Mmap(fd int, offset int64, length int, prot int, flags int) (data []byte, e
 	data, err = syscall.Mmap(fd, offset, length, prot, flags)
 	if err != nil {
 		atomic.AddUint64(&mapCount, ^uint64(0)) // decrement
+		if strings.Contains(err.Error(), "cannot allocate memory") {
+			err = errors.New("mmap 'cannot allocate memory' — please see the troubleshooting how-to in the FeatureBase docs.")
+		}
 	}
 	return data, err
 }

From 5324431ab049dfeece400b1716347731f114d66a Mon Sep 17 00:00:00 2001
From: Samir Patel <48686912+54mir@users.noreply.github.com>
Date: Thu, 16 Sep 2021 16:12:57 -0500
Subject: [PATCH 21/66] Rename pilosa to featurebase

---
 .circleci/config.yml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 0ae2f1f43..30b2ef5e4 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -35,7 +35,7 @@ commands:
       - checkout
       - restore-mod-cache
   skip-if-root-unchanged:
-    description: "skips the parent job if the PR includes no changes to pilosa"
+    description: "skips the parent job if the PR includes no changes to featurebase"
     steps:
       - run: |
           ROOT_CHANGED_FILES="$(git diff --name-only HEAD $(git merge-base master HEAD) | grep -v '^lattice/')" || true
@@ -202,7 +202,7 @@ jobs:
           version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
       - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
       - run: make docker GO_VERSION=1.15.8
-      - run: docker run pilosa:$(git describe --tags) help
+      - run: docker run featurebase:$(git describe --tags) help
   dockerhub-upload-unstable:
     executor:
       name: golang
@@ -212,8 +212,8 @@ jobs:
           version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
       - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
       - run: make docker
-      - run: docker run pilosa:$(git describe --tags) help
-      - run: make docker-tag-push DOCKER_TARGET=moleculacorp/pilosa:<< pipeline.git.branch >>
+      - run: docker run featurebase:$(git describe --tags) help
+      - run: make docker-tag-push DOCKER_TARGET=moleculacorp/featurebase:<< pipeline.git.branch >>
   dockerhub-upload-stable:
     executor:
       name: golang
@@ -223,9 +223,9 @@ jobs:
           version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
       - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
       - run: make docker
-      - run: docker run pilosa:$(git describe --tags) help
-      - run: make docker-tag-push DOCKER_TARGET=moleculacorp/pilosa:<< pipeline.git.tag >>
-      - run: make docker-tag-push DOCKER_TARGET=moleculacorp/pilosa:latest
+      - run: docker run featurebase:$(git describe --tags) help
+      - run: make docker-tag-push DOCKER_TARGET=moleculacorp/featurebase:<< pipeline.git.tag >>
+      - run: make docker-tag-push DOCKER_TARGET=moleculacorp/featurebase:latest
 
 workflows:
   build:

From ff8f0cab966c01829547c9e84f22a3c213065011 Mon Sep 17 00:00:00 2001
From: Ben Johnson <benbjohnson@yahoo.com>
Date: Fri, 24 Sep 2021 14:02:15 -0600
Subject: [PATCH 22/66] Add SQL type checking

---
 planner.go       | 447 +++++++++++++++++++--------
 planner_test.go  |  99 ++++--
 sql2/ast.go      |  70 ++++-
 sql2/ast_test.go |   4 +-
 sql2/walk.go     | 785 +++++++++++++++++++++++++++++------------------
 5 files changed, 955 insertions(+), 450 deletions(-)

diff --git a/planner.go b/planner.go
index 4b080d279..9fd141fc8 100644
--- a/planner.go
+++ b/planner.go
@@ -42,6 +42,10 @@ func (p *Planner) PlanStatement(ctx context.Context, stmt sql2.Statement) (*Stmt
 }
 
 func (p *Planner) planStatement(ctx context.Context, stmt sql2.Statement) (StmtNode, error) {
+	if err := p.checkStatement(stmt); err != nil {
+		return nil, err
+	}
+
 	switch stmt := stmt.(type) {
 	case *sql2.SelectStatement:
 		return p.planSelectStatement(ctx, stmt)
@@ -58,21 +62,10 @@ func (p *Planner) planSelectStatement(ctx context.Context, stmt *sql2.SelectStat
 }
 
 func (p *Planner) planAggregateSelectStatement(ctx context.Context, stmt *sql2.SelectStatement) (_ StmtNode, err error) {
-	// Extract table name from source.
-	var source *sql2.QualifiedTableName
-	switch src := stmt.Source.(type) {
-	case *sql2.JoinClause:
-		return nil, fmt.Errorf("cannot use JOIN in aggregate query")
-	case *sql2.ParenSource:
-		return nil, fmt.Errorf("cannot use parenthesized source in aggregate query")
-	case *sql2.QualifiedTableName:
-		source = src
-	case *sql2.SelectStatement:
-		return nil, fmt.Errorf("cannot use sub-select in aggregate query")
-	default:
-		return nil, fmt.Errorf("unexpected source type in aggregate query: %T", source)
+	indexName, err := statementTableName(stmt)
+	if err != nil {
+		return nil, err
 	}
-	indexName := sql2.IdentName(source.Name)
 
 	// Convert WHERE clause.
 	cond, err := p.planExprPQL(ctx, stmt, stmt.WhereExpr)
@@ -83,16 +76,18 @@ func (p *Planner) planAggregateSelectStatement(ctx context.Context, stmt *sql2.S
 	// Extract calls and grouped expressions from column list.
 	// TODO: Recursively traverse all expression trees.
 	var calls []*sql2.Call
-	var aliases []string
-	// var groupByCols []*sql2.Ident // TODO: Convert to QualifiedRef
+	var columns []*StmtColumn
 	for _, c := range stmt.Columns {
-		aliases = append(aliases, c.Name())
+		columns = append(columns, &StmtColumn{
+			Name: c.Name(),
+			Type: sql2.ExprDataType(c.Expr),
+		})
 
 		switch c := c.Expr.(type) {
 		case *sql2.Call:
 			calls = append(calls, c)
-		case *sql2.Ident:
-			// groupByCols = append(groupByCols, c)
+		case *sql2.QualifiedRef:
+			// allowed
 		default:
 			return nil, fmt.Errorf("unsupported expression type in aggregate query: %T", c)
 		}
@@ -107,8 +102,8 @@ func (p *Planner) planAggregateSelectStatement(ctx context.Context, stmt *sql2.S
 	var groupByColNames []string
 	for _, expr := range stmt.GroupByExprs {
 		switch expr := expr.(type) {
-		case *sql2.Ident:
-			groupByColNames = append(groupByColNames, expr.Name)
+		case *sql2.QualifiedRef:
+			groupByColNames = append(groupByColNames, expr.Column.Name)
 		default:
 			return nil, fmt.Errorf("unsupported expression type in GROUP BY clause: %T", expr)
 		}
@@ -119,7 +114,7 @@ func (p *Planner) planAggregateSelectStatement(ctx context.Context, stmt *sql2.S
 	switch callName {
 	case "COUNT":
 		if len(groupByColNames) == 0 {
-			return NewCountNode(p.executor, indexName, aliases[0], cond), nil
+			return NewCountNode(p.executor, indexName, columns[0], cond), nil
 		}
 
 		var aggregate *pql.Call
@@ -127,7 +122,7 @@ func (p *Planner) planAggregateSelectStatement(ctx context.Context, stmt *sql2.S
 			if len(calls[0].Args) != 1 {
 				return nil, fmt.Errorf("distinct count must have exactly one field specified")
 			}
-			field, ok := calls[0].Args[0].(*sql2.Ident)
+			ref, ok := calls[0].Args[0].(*sql2.QualifiedRef)
 			if !ok {
 				return nil, fmt.Errorf("distinct count argument must be a field name")
 			}
@@ -136,28 +131,28 @@ func (p *Planner) planAggregateSelectStatement(ctx context.Context, stmt *sql2.S
 				Name: "Count",
 				Children: []*pql.Call{{
 					Name: "Distinct",
-					Args: map[string]interface{}{"field": field.Name},
+					Args: map[string]interface{}{"field": ref.Column.Name},
 				}},
 			}
 		}
 
-		return NewGroupByNode(p.executor, indexName, groupByColNames, aliases, aggregate, cond), nil
+		return NewGroupByNode(p.executor, indexName, groupByColNames, columns, aggregate, cond), nil
 
 	case "SUM":
 		if len(calls[0].Args) != 1 {
 			return nil, fmt.Errorf("sum must have exactly one field specified")
 		}
-		field, ok := calls[0].Args[0].(*sql2.Ident)
+		ref, ok := calls[0].Args[0].(*sql2.QualifiedRef)
 		if !ok {
 			return nil, fmt.Errorf("sum argument must be a field name")
 		}
 
 		aggregate := &pql.Call{
 			Name: "Sum",
-			Args: map[string]interface{}{"field": field.Name},
+			Args: map[string]interface{}{"field": ref.Column.Name},
 		}
 
-		return NewGroupByNode(p.executor, indexName, groupByColNames, aliases, aggregate, cond), nil
+		return NewGroupByNode(p.executor, indexName, groupByColNames, columns, aggregate, cond), nil
 
 	default:
 		return nil, fmt.Errorf("unsupported call in aggregate query: %s", callName)
@@ -167,21 +162,10 @@ func (p *Planner) planAggregateSelectStatement(ctx context.Context, stmt *sql2.S
 }
 
 func (p *Planner) planNonAggregateSelectStatement(ctx context.Context, stmt *sql2.SelectStatement) (_ StmtNode, err error) {
-	// Extract table name from source.
-	var source *sql2.QualifiedTableName
-	switch src := stmt.Source.(type) {
-	case *sql2.JoinClause:
-		return nil, fmt.Errorf("cannot use JOIN in non-aggregate query")
-	case *sql2.ParenSource:
-		return nil, fmt.Errorf("cannot use parenthesized source in non-aggregate query")
-	case *sql2.QualifiedTableName:
-		source = src
-	case *sql2.SelectStatement:
-		return nil, fmt.Errorf("cannot use sub-select in non-aggregate query")
-	default:
-		return nil, fmt.Errorf("unexpected source type in non-aggregate query: %T", source)
+	indexName, err := statementTableName(stmt)
+	if err != nil {
+		return nil, err
 	}
-	indexName := sql2.IdentName(source.Name)
 
 	// Lookup index.
 	idx := p.executor.Holder.Index(indexName)
@@ -196,57 +180,24 @@ func (p *Planner) planNonAggregateSelectStatement(ctx context.Context, stmt *sql
 	}
 
 	// Build column list.
-	var columnNames, columnAliases []string
+	var srcs []string
+	var columns []*StmtColumn
 	for _, col := range stmt.Columns {
-		// Unqualified wildcard.
-		if col.Star.IsValid() {
-			columnNames = append(columnNames, "_id")
-			columnAliases = append(columnAliases, "_id")
-
-			for _, field := range idx.Fields() {
-				if field.Name() == "_exists" {
-					continue
-				}
-				columnNames = append(columnNames, field.Name())
-				columnAliases = append(columnAliases, field.Name())
-			}
-			continue
-		}
-
 		// Handle expressions and qualified references.
 		switch expr := col.Expr.(type) {
-		case *sql2.Ident:
-			columnNames = append(columnNames, expr.Name)
-			columnAliases = append(columnAliases, col.Name())
-
 		case *sql2.QualifiedRef:
-			if tbl := sql2.IdentName(expr.Table); tbl != "" && tbl != source.TableName() {
-				return nil, fmt.Errorf("no such table: %q", tbl)
-			}
-
-			if expr.Star.IsValid() {
-				columnNames = append(columnNames, "_id")
-				columnAliases = append(columnAliases, "_id")
-
-				for _, field := range idx.Fields() {
-					if field.Name() == "_exists" {
-						continue
-					}
-					columnNames = append(columnNames, field.Name())
-					columnAliases = append(columnAliases, field.Name())
-				}
-
-			} else {
-				columnNames = append(columnNames, sql2.IdentName(expr.Column))
-				columnAliases = append(columnAliases, sql2.IdentName(expr.Column))
-			}
+			srcs = append(srcs, sql2.IdentName(expr.Column))
+			columns = append(columns, &StmtColumn{
+				Name: sql2.IdentName(expr.Column),
+				Type: sql2.ExprDataType(col.Expr),
+			})
 
 		default:
 			return nil, fmt.Errorf("unsupported column expression: %T", expr)
 		}
 	}
 
-	return NewExtractNode(p.executor, indexName, columnNames, columnAliases, cond), nil
+	return NewExtractNode(p.executor, indexName, srcs, columns, cond), nil
 }
 
 // planExprPQL returns a PQL call tree for a given expression.
@@ -322,8 +273,8 @@ func (p *Planner) planBinaryExprPQL(ctx context.Context, stmt *sql2.SelectStatem
 	case sql2.EQ, sql2.NE, sql2.LT, sql2.LE, sql2.GT, sql2.GE:
 		// Ensure field reference exists in binary expression.
 		x, y := expr.X, expr.Y
-		xIdent, xOk := x.(*sql2.Ident)
-		yIdent, yOk := y.(*sql2.Ident)
+		xRef, xOk := x.(*sql2.QualifiedRef)
+		yRef, yOk := y.(*sql2.QualifiedRef)
 		if xOk && yOk {
 			return nil, fmt.Errorf("cannot compare fields in a WHERE clause")
 		} else if !xOk && !yOk {
@@ -332,7 +283,7 @@ func (p *Planner) planBinaryExprPQL(ctx context.Context, stmt *sql2.SelectStatem
 
 		// Rewrite expression so field ref is LHS.
 		if !xOk && yOk {
-			xIdent, y = yIdent, x
+			xRef, y = yRef, x
 			switch op {
 			case sql2.LT:
 				op = sql2.GT
@@ -355,7 +306,7 @@ func (p *Planner) planBinaryExprPQL(ctx context.Context, stmt *sql2.SelectStatem
 			return &pql.Call{
 				Name: "Row",
 				Args: map[string]interface{}{
-					sql2.IdentName(xIdent): pqlValue,
+					sql2.IdentName(xRef.Column): pqlValue,
 				},
 			}, nil
 		}
@@ -367,7 +318,7 @@ func (p *Planner) planBinaryExprPQL(ctx context.Context, stmt *sql2.SelectStatem
 		return &pql.Call{
 			Name: "Row",
 			Args: map[string]interface{}{
-				sql2.IdentName(xIdent): &pql.Condition{
+				sql2.IdentName(xRef.Column): &pql.Condition{
 					Op:    pqlOp,
 					Value: pqlValue,
 				},
@@ -426,6 +377,221 @@ func sqlToPQLValue(expr sql2.Expr) (interface{}, error) {
 	}
 }
 
+func (p *Planner) checkStatement(stmt sql2.Statement) error {
+	switch stmt := stmt.(type) {
+	case *sql2.SelectStatement:
+		return p.checkSelectStatement(stmt)
+	default:
+		return nil
+	}
+}
+
+func (p *Planner) checkSelectStatement(stmt *sql2.SelectStatement) error {
+	indexName, err := statementTableName(stmt)
+	if err != nil {
+		return err
+	}
+
+	// Look up index.
+	idx := p.executor.Holder.Index(indexName)
+	if idx == nil {
+		return newNotFoundError(ErrIndexNotFound, indexName)
+	}
+
+	// Replace wildcards with column references.
+	columns := make([]*sql2.ResultColumn, 0, len(stmt.Columns))
+	for _, col := range stmt.Columns {
+		// Unqualified wildcard.
+		isWildcard := col.Star.IsValid()
+		if ref, ok := col.Expr.(*sql2.QualifiedRef); ok && ref.Star.IsValid() {
+			if ref.Table.Name != indexName {
+				return fmt.Errorf("no such table: %q", ref.Table.Name)
+			}
+			isWildcard = true
+		}
+
+		// Simply add column as-is if it is not a wildcard.
+		if !isWildcard {
+			columns = append(columns, col)
+			continue
+		}
+
+		// Add identifier field first.
+		columns = append(columns, &sql2.ResultColumn{
+			Expr: &sql2.QualifiedRef{
+				Table:  &sql2.Ident{Name: idx.Name()},
+				Column: &sql2.Ident{Name: "_id"},
+			},
+		})
+
+		// Then add all fields besides the existence bit.
+		for _, field := range idx.Fields() {
+			if field.Name() == "_exists" {
+				continue
+			}
+			columns = append(columns, &sql2.ResultColumn{
+				Expr: &sql2.QualifiedRef{
+					Table:  &sql2.Ident{Name: idx.Name()},
+					Column: &sql2.Ident{Name: field.Name()},
+				},
+			})
+		}
+	}
+	stmt.Columns = columns
+
+	// Type check expressions in statement.
+	for _, col := range stmt.Columns {
+		if err := p.checkExpr(&col.Expr, stmt); err != nil {
+			return err
+		}
+	}
+
+	if err := p.checkExpr(&stmt.WhereExpr, stmt); err != nil {
+		return err
+	}
+
+	for i := range stmt.GroupByExprs {
+		if err := p.checkExpr(&stmt.GroupByExprs[i], stmt); err != nil {
+			return err
+		}
+	}
+
+	if err := p.checkExpr(&stmt.HavingExpr, stmt); err != nil {
+		return err
+	}
+
+	for _, term := range stmt.OrderingTerms {
+		if err := p.checkExpr(&term.X, stmt); err != nil {
+			return err
+		}
+	}
+
+	if err := p.checkExpr(&stmt.LimitExpr, stmt); err != nil {
+		return err
+	}
+
+	if err := p.checkExpr(&stmt.OffsetExpr, stmt); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (p *Planner) checkExpr(expr *sql2.Expr, stmt sql2.Statement) error {
+	if e, err := sql2.Walk(&sqlExprTypeChecker{
+		holder: p.executor.Holder,
+		stmt:   stmt,
+	}, *expr); err != nil {
+		return err
+	} else if e != nil {
+		*expr = e.(sql2.Expr)
+	} else {
+		*expr = nil
+	}
+	return nil
+}
+
+// sqlExprTypeChecker recursively performs type checking within an expression.
+// Called by sqlTypeChecker. Implements sql2.Visitor.
+type sqlExprTypeChecker struct {
+	holder *Holder
+	stmt   sql2.Statement // scope
+}
+
+var _ sql2.Visitor = (*sqlExprTypeChecker)(nil)
+
+func (v *sqlExprTypeChecker) Visit(node sql2.Node) (_ sql2.Visitor, _ sql2.Node, err error) {
+	switch n := node.(type) {
+	case *sql2.Call:
+		for i := range n.Args {
+			if err := v.checkExpr(&n.Args[i]); err != nil {
+				return nil, nil, err
+			}
+		}
+		return nil, node, nil // skip
+	case *sql2.Ident:
+		if node, err = v.visitIdent(n); err != nil {
+			return nil, nil, err
+		}
+		return nil, node, nil
+	case *sql2.QualifiedRef:
+		if node, err = v.visitQualifiedRef(n); err != nil {
+			return nil, nil, err
+		}
+		return nil, node, nil
+	default:
+		return v, node, nil
+	}
+}
+
+func (v *sqlExprTypeChecker) visitIdent(ident *sql2.Ident) (sql2.Node, error) {
+	indexName, err := statementTableName(v.stmt)
+	if err != nil {
+		return nil, err
+	}
+
+	// Convert to a table qualified reference and validate through ref visit function.
+	return v.visitQualifiedRef(&sql2.QualifiedRef{
+		Table:  &sql2.Ident{Name: indexName},
+		Column: &sql2.Ident{Name: ident.Name},
+	})
+}
+
+func (v *sqlExprTypeChecker) visitQualifiedRef(ref *sql2.QualifiedRef) (sql2.Node, error) {
+	idx := v.holder.Index(ref.Table.Name)
+	if idx == nil {
+		return nil, newNotFoundError(ErrIndexNotFound, ref.Table.Name)
+	}
+
+	switch name := ref.Column.Name; name {
+	case "_id":
+		ref.DataType = sql2.DataTypeInt
+	default:
+		field := idx.Field(ref.Column.Name)
+		if field == nil {
+			return nil, newNotFoundError(ErrFieldNotFound, ref.Column.Name)
+		}
+		ref.DataType = fieldSQLDataType(field)
+	}
+
+	return ref, nil
+}
+
+func (v *sqlExprTypeChecker) checkExpr(node *sql2.Expr) error {
+	if expr, err := sql2.Walk(&sqlExprTypeChecker{
+		holder: v.holder,
+		stmt:   v.stmt,
+	}, *node); err != nil {
+		return err
+	} else if expr != nil {
+		*node = expr.(sql2.Expr)
+	} else {
+		*node = nil
+	}
+	return nil
+}
+
+func (v *sqlExprTypeChecker) VisitEnd(node sql2.Node) (sql2.Node, error) { return node, nil }
+
+func fieldSQLDataType(f *Field) string {
+	if f.Keys() {
+		return sql2.DataTypeText
+	}
+
+	switch f.Type() {
+	case FieldTypeInt, FieldTypeMutex, FieldTypeSet:
+		return sql2.DataTypeInt
+	case FieldTypeBool:
+		return sql2.DataTypeBool
+	case FieldTypeDecimal:
+		return sql2.DataTypeDecimal
+	case FieldTypeTime, FieldTypeTimestamp:
+		return sql2.DataTypeTimestamp
+	default:
+		return ""
+	}
+}
+
 type Stmt struct {
 	node StmtNode
 }
@@ -473,7 +639,7 @@ func (rs *StmtRows) Err() error {
 	return nil
 }
 
-func (rs *StmtRows) Columns() []string {
+func (rs *StmtRows) Columns() []*StmtColumn {
 	return rs.node.Columns()
 }
 
@@ -574,6 +740,11 @@ func (r *StmtRow) Err() error {
 	return r.err
 }
 
+type StmtColumn struct {
+	Name string
+	Type string
+}
+
 type StmtNode interface {
 	// Initializes the node to its start.
 	First(ctx context.Context) error
@@ -585,7 +756,7 @@ type StmtNode interface {
 	Row() []interface{}
 
 	// Returns column definitions for the node.
-	Columns() []string
+	Columns() []*StmtColumn
 
 	// Returns a reference to the value register for a named column.
 	// Lookup(table, column string) (interface{}, error)
@@ -597,39 +768,38 @@ var _ StmtNode = (*ExtractNode)(nil)
 type ExtractNode struct {
 	executor  *executor
 	indexName string
-	columns   []string
-	aliases   []string
+	srcs      []string
+	columns   []*StmtColumn
 	cond      *pql.Call
 
 	result []ExtractedTableColumn
 	row    []interface{}
 }
 
-func NewExtractNode(executor *executor, indexName string, columns, aliases []string, cond *pql.Call) *ExtractNode {
+func NewExtractNode(executor *executor, indexName string, srcs []string, columns []*StmtColumn, cond *pql.Call) *ExtractNode {
 	if cond == nil {
 		cond = &pql.Call{Name: "All"}
 	}
 
 	// Ensure ID column is always the first column.
-	if len(columns) > 0 && columns[0] != "_id" {
-		columns = append([]string{"_id"}, columns...)
-		aliases = append([]string{"_id"}, aliases...)
+	// TODO(benbjohnson): Don't require id first.
+	if len(srcs) > 0 && srcs[0] != "_id" {
+		srcs = append([]string{"_id"}, srcs...)
+		columns = append([]*StmtColumn{{Name: "_id", Type: sql2.DataTypeInt}}, columns...)
 	}
 
-	// TODO: Move "_id" column to the first position if it is specified later on in column list.
-
 	return &ExtractNode{
 		executor:  executor,
 		indexName: indexName,
-		columns:   columns, // source column names
-		aliases:   aliases, // external column alias
+		srcs:      srcs,    // source column names
+		columns:   columns, // external column alias
 		cond:      cond,
-		row:       make([]interface{}, len(columns)),
+		row:       make([]interface{}, len(srcs)),
 	}
 }
 
-func (n *ExtractNode) Columns() []string {
-	return n.aliases
+func (n *ExtractNode) Columns() []*StmtColumn {
+	return n.columns
 }
 
 func (n *ExtractNode) First(ctx context.Context) error {
@@ -674,11 +844,11 @@ func (n *ExtractNode) init(ctx context.Context) error {
 	// Generate PQL query with all specified rows.
 	// Skip first column as it is the ID column.
 	call := &pql.Call{Name: "Extract", Children: []*pql.Call{n.cond}}
-	for _, column := range n.columns[1:] {
+	for _, src := range n.srcs[1:] {
 		call.Children = append(call.Children,
 			&pql.Call{
 				Name: "Rows",
-				Args: map[string]interface{}{"field": column},
+				Args: map[string]interface{}{"field": src},
 			},
 		)
 	}
@@ -709,28 +879,28 @@ var _ StmtNode = (*CountNode)(nil)
 
 // CountNode executes a COUNT(*) against a FeatureBase index and returns a single row.
 type CountNode struct {
-	executor   *executor
-	indexName  string
-	columnName string
-	cond       *pql.Call // conditional
+	executor  *executor
+	indexName string
+	column    *StmtColumn
+	cond      *pql.Call // conditional
 
 	row []interface{}
 }
 
-func NewCountNode(executor *executor, indexName string, columnName string, cond *pql.Call) *CountNode {
+func NewCountNode(executor *executor, indexName string, column *StmtColumn, cond *pql.Call) *CountNode {
 	if cond == nil {
 		cond = &pql.Call{Name: "All"}
 	}
 	return &CountNode{
-		executor:   executor,
-		indexName:  indexName,
-		columnName: columnName,
-		cond:       cond,
+		executor:  executor,
+		indexName: indexName,
+		column:    column,
+		cond:      cond,
 	}
 }
 
-func (n *CountNode) Columns() []string {
-	return []string{n.columnName}
+func (n *CountNode) Columns() []*StmtColumn {
+	return []*StmtColumn{n.column}
 }
 
 func (n *CountNode) First(ctx context.Context) error {
@@ -764,8 +934,8 @@ func (n *CountNode) Row() []interface{} { return n.row }
 type GroupByNode struct {
 	executor  *executor
 	indexName string
-	columns   []string
-	aliases   []string
+	srcs      []string
+	columns   []*StmtColumn
 	aggregate *pql.Call
 	cond      *pql.Call
 
@@ -775,20 +945,20 @@ type GroupByNode struct {
 	row []interface{}
 }
 
-func NewGroupByNode(executor *executor, indexName string, columns, aliases []string, aggregate, cond *pql.Call) *GroupByNode {
+func NewGroupByNode(executor *executor, indexName string, srcs []string, columns []*StmtColumn, aggregate, cond *pql.Call) *GroupByNode {
 	return &GroupByNode{
 		executor:  executor,
 		indexName: indexName,
+		srcs:      srcs,
 		columns:   columns,
-		aliases:   aliases,
 		aggregate: aggregate,
 		cond:      cond,
-		row:       make([]interface{}, len(columns)+1),
+		row:       make([]interface{}, len(srcs)+1),
 	}
 }
 
-func (n *GroupByNode) Columns() []string {
-	return append([]string{"_aggregate"}, n.columns...)
+func (n *GroupByNode) Columns() []*StmtColumn {
+	return n.columns
 }
 
 func (n *GroupByNode) First(ctx context.Context) error {
@@ -840,9 +1010,9 @@ func (n *GroupByNode) fetch(ctx context.Context) (*GroupCounts, error) {
 	}
 
 	// Choose fields to group by.
-	for _, col := range n.columns {
+	for _, src := range n.srcs {
 		call.Children = append(call.Children, &pql.Call{
-			Name: "Rows", Args: map[string]interface{}{"_field": col},
+			Name: "Rows", Args: map[string]interface{}{"_field": src},
 		})
 	}
 
@@ -862,3 +1032,30 @@ func (n *GroupByNode) fetch(ctx context.Context) (*GroupCounts, error) {
 }
 
 func (n *GroupByNode) Row() []interface{} { return n.row }
+
+// statementTableName returns the table name for a single table SELECT statement.
+//
+// NOTE: This function is only temporary until we support more source types.
+func statementTableName(stmt sql2.Statement) (string, error) {
+	switch stmt := stmt.(type) {
+	case *sql2.SelectStatement:
+		return sourceTableName(stmt.Source)
+	default:
+		return "", fmt.Errorf("statement not currently supported")
+	}
+}
+
+func sourceTableName(source sql2.Source) (string, error) {
+	switch source := source.(type) {
+	case *sql2.JoinClause:
+		return "", fmt.Errorf("joins are not currently supported")
+	case *sql2.ParenSource:
+		return "", fmt.Errorf("parenthesized source is not currently supported")
+	case *sql2.QualifiedTableName:
+		return sql2.IdentName(source.Name), nil
+	case *sql2.SelectStatement:
+		return "", fmt.Errorf("sub-selects are not currently supported")
+	default:
+		return "", fmt.Errorf("unexpected source type: %T", source)
+	}
+}
diff --git a/planner_test.go b/planner_test.go
index 4748046b6..e92bd77a6 100644
--- a/planner_test.go
+++ b/planner_test.go
@@ -210,54 +210,80 @@ func TestPlanner_Select(t *testing.T) {
 	}
 
 	t.Run("UnqualifiedColumns", func(t *testing.T) {
-		results := mustQueryRows(t, c.GetNode(0).Server, `SELECT _id, a, b FROM i0`)
-		if diff := cmp.Diff(results, [][]interface{}{
+		results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT _id, a, b FROM i0`)
+		if diff := cmp.Diff([][]interface{}{
 			{int64(1), int64(10), int64(100)},
 			{int64(2), int64(20), int64(200)},
-		}); diff != "" {
+		}, results); diff != "" {
+			t.Fatal(diff)
+		}
+
+		if diff := cmp.Diff([]*pilosa.StmtColumn{
+			{Name: "_id", Type: "INT"},
+			{Name: "a", Type: "INT"},
+			{Name: "b", Type: "INT"},
+		}, columns); diff != "" {
 			t.Fatal(diff)
 		}
 	})
 
 	t.Run("QualifiedColumns", func(t *testing.T) {
-		results := mustQueryRows(t, c.GetNode(0).Server, `SELECT i0._id, i0.a, i0.b FROM i0`)
-		if diff := cmp.Diff(results, [][]interface{}{
+		results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT i0._id, i0.a, i0.b FROM i0`)
+		if diff := cmp.Diff([][]interface{}{
 			{int64(1), int64(10), int64(100)},
 			{int64(2), int64(20), int64(200)},
-		}); diff != "" {
+		}, results); diff != "" {
+			t.Fatal(diff)
+		}
+
+		if diff := cmp.Diff([]*pilosa.StmtColumn{
+			{Name: "_id", Type: "INT"},
+			{Name: "a", Type: "INT"},
+			{Name: "b", Type: "INT"},
+		}, columns); diff != "" {
 			t.Fatal(diff)
 		}
 	})
 
 	t.Run("UnqualifiedStar", func(t *testing.T) {
-		results := mustQueryRows(t, c.GetNode(0).Server, `SELECT * FROM i0`)
-		if diff := cmp.Diff(results, [][]interface{}{
+		results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT * FROM i0`)
+		if diff := cmp.Diff([][]interface{}{
 			{int64(1), int64(10), int64(100)},
 			{int64(2), int64(20), int64(200)},
-		}); diff != "" {
+		}, results); diff != "" {
+			t.Fatal(diff)
+		}
+
+		if diff := cmp.Diff([]*pilosa.StmtColumn{
+			{Name: "_id", Type: "INT"},
+			{Name: "a", Type: "INT"},
+			{Name: "b", Type: "INT"},
+		}, columns); diff != "" {
 			t.Fatal(diff)
 		}
 	})
 
 	t.Run("QualifiedStar", func(t *testing.T) {
-		results := mustQueryRows(t, c.GetNode(0).Server, `SELECT i0.* FROM i0`)
-		if diff := cmp.Diff(results, [][]interface{}{
+		results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT i0.* FROM i0`)
+		if diff := cmp.Diff([][]interface{}{
 			{int64(1), int64(10), int64(100)},
 			{int64(2), int64(20), int64(200)},
-		}); diff != "" {
+		}, results); diff != "" {
+			t.Fatal(diff)
+		}
+
+		if diff := cmp.Diff([]*pilosa.StmtColumn{
+			{Name: "_id", Type: "INT"},
+			{Name: "a", Type: "INT"},
+			{Name: "b", Type: "INT"},
+		}, columns); diff != "" {
 			t.Fatal(diff)
 		}
 	})
 
 	t.Run("ErrFieldNotFound", func(t *testing.T) {
-		stmt, err := c.GetNode(0).Server.PlanSQL(context.Background(), `SELECT xyz FROM i0`)
-		if err != nil {
-			t.Fatal(err)
-		}
-		defer stmt.Close()
-
-		var xyz interface{}
-		if err := stmt.QueryRowContext(context.Background()).Scan(&xyz); err == nil || !strings.Contains(err.Error(), `xyz: field not found`) {
+		_, err := c.GetNode(0).Server.PlanSQL(context.Background(), `SELECT xyz FROM i0`)
+		if err == nil || !strings.Contains(err.Error(), `xyz: field not found`) {
 			t.Fatalf("unexpected error: %v", err)
 		}
 	})
@@ -301,37 +327,58 @@ func TestPlanner_GroupBy(t *testing.T) {
 	}
 
 	t.Run("Count", func(t *testing.T) {
-		results := mustQueryRows(t, c.GetNode(0).Server, `SELECT COUNT(*), x FROM i0 GROUP BY x`)
+		results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT COUNT(*), x FROM i0 GROUP BY x`)
 		if diff := cmp.Diff([][]interface{}{
 			{int64(2), int64(10)},
 			{int64(2), int64(20)},
 		}, results); diff != "" {
 			t.Fatal(diff)
 		}
+
+		if diff := cmp.Diff([]*pilosa.StmtColumn{
+			{Name: "count", Type: "INT"},
+			{Name: "x", Type: "INT"},
+		}, columns); diff != "" {
+			t.Fatal(diff)
+		}
 	})
 
 	t.Run("DistinctCount", func(t *testing.T) {
-		results := mustQueryRows(t, c.GetNode(0).Server, `SELECT COUNT(DISTINCT z), x FROM i0 GROUP BY x`)
+		results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT COUNT(DISTINCT z), x FROM i0 GROUP BY x`)
 		if diff := cmp.Diff([][]interface{}{
 			{int64(1), int64(10)},
 			{int64(2), int64(20)},
 		}, results); diff != "" {
 			t.Fatal(diff)
 		}
+
+		if diff := cmp.Diff([]*pilosa.StmtColumn{
+			{Name: "count", Type: "INT"},
+			{Name: "x", Type: "INT"},
+		}, columns); diff != "" {
+			t.Fatal(diff)
+		}
 	})
 
 	t.Run("Sum", func(t *testing.T) {
-		results := mustQueryRows(t, c.GetNode(0).Server, `SELECT sum(y), x FROM i0 GROUP BY x`)
+		results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT sum(y), x FROM i0 GROUP BY x`)
 		if diff := cmp.Diff([][]interface{}{
 			{int64(300), int64(10)},
 			{int64(100), int64(20)},
 		}, results); diff != "" {
 			t.Fatal(diff)
 		}
+
+		if diff := cmp.Diff([]*pilosa.StmtColumn{
+			{Name: "sum", Type: "INT"},
+			{Name: "x", Type: "INT"},
+		}, columns); diff != "" {
+			t.Fatal(diff)
+		}
 	})
 }
 
-func mustQueryRows(tb testing.TB, svr *pilosa.Server, q string) [][]interface{} {
+func mustQueryRows(tb testing.TB, svr *pilosa.Server, q string) (results [][]interface{}, columns []*pilosa.StmtColumn) {
 	tb.Helper()
 
 	stmt, err := svr.PlanSQL(context.Background(), q)
@@ -345,7 +392,7 @@ func mustQueryRows(tb testing.TB, svr *pilosa.Server, q string) [][]interface{}
 		tb.Fatal(err)
 	}
 
-	results := make([][]interface{}, 0)
+	results = make([][]interface{}, 0)
 	for rows.Next() {
 		result := make([]interface{}, len(rows.Columns()))
 
@@ -365,5 +412,5 @@ func mustQueryRows(tb testing.TB, svr *pilosa.Server, q string) [][]interface{}
 		tb.Fatal(err)
 	}
 
-	return results
+	return results, rows.Columns()
 }
diff --git a/sql2/ast.go b/sql2/ast.go
index ce231781f..929d1f945 100644
--- a/sql2/ast.go
+++ b/sql2/ast.go
@@ -194,6 +194,26 @@ func StatementSource(stmt Statement) Source {
 	}
 }
 
+// Data types
+const (
+	DataTypeBool      = "BOOL"
+	DataTypeDecimal   = "DECIMAL"
+	DataTypeInt       = "INT"
+	DataTypeSet       = "SET"
+	DataTypeText      = "TEXT"
+	DataTypeTimestamp = "TIMESTAMP"
+)
+
+// IsDataTypeValid returns true if typ is a valid data type.
+func IsDataTypeValid(typ string) bool {
+	switch typ {
+	case DataTypeBool, DataTypeInt, DataTypeDecimal, DataTypeText:
+		return true
+	default:
+		return false
+	}
+}
+
 type Expr interface {
 	Node
 	expr()
@@ -279,6 +299,49 @@ func cloneExprs(a []Expr) []Expr {
 	return other
 }
 
+// ExprDataType returns the data type for an expression.
+func ExprDataType(expr Expr) string {
+	if expr == nil {
+		return ""
+	}
+
+	switch expr := expr.(type) {
+	// Simple type assertions
+	case *BindExpr, *ExprList, *Ident, *NullLit, *Raise:
+		return ""
+	case *BlobLit, *StringLit:
+		return DataTypeText
+	case *BoolLit, *Exists, *Range:
+		return DataTypeBool
+	case *NumberLit:
+		return DataTypeInt
+
+	// Complex type assertions
+	case *BinaryExpr:
+		return ExprDataType(expr.X)
+	case *Call:
+		return DataTypeInt // TODO: May be different for some aggregations
+	case *CaseExpr:
+		if len(expr.Blocks) > 0 {
+			return ExprDataType(expr.Blocks[0].Body)
+		} else if expr.ElseExpr != nil {
+			return ExprDataType(expr.ElseExpr)
+		}
+		return ""
+	case *CastExpr:
+		return "" // TODO: Inspect expr.Type.Name
+	case *ParenExpr:
+		return ExprDataType(expr.X)
+	case *QualifiedRef:
+		return expr.DataType
+	case *UnaryExpr:
+		return ExprDataType(expr.X)
+
+	default:
+		panic(fmt.Sprintf("invalid expr type: %T", expr))
+	}
+}
+
 // ExprString returns the string representation of expr.
 // Returns a blank string if expr is nil.
 func ExprString(expr Expr) string {
@@ -1811,6 +1874,9 @@ type QualifiedRef struct {
 	Dot    Pos    // position of dot
 	Star   Pos    // position of * (result column only)
 	Column *Ident // column name
+
+	// Set by the planner; not at parse-time
+	DataType string
 }
 
 // IsAggregate returns false.
@@ -3061,10 +3127,12 @@ func (c *ResultColumn) Name() string {
 	}
 
 	switch expr := c.Expr.(type) {
+	case *Call:
+		return strings.ToLower(IdentName(expr.Name))
 	case *Ident:
 		return IdentName(expr)
 	case *QualifiedRef:
-		return expr.String()
+		return IdentName(expr.Column)
 	default:
 		return ""
 	}
diff --git a/sql2/ast_test.go b/sql2/ast_test.go
index 8d3c1fed5..25cf95ac9 100644
--- a/sql2/ast_test.go
+++ b/sql2/ast_test.go
@@ -1143,14 +1143,14 @@ func AssertNodeStringerPanic(tb testing.TB, node sql.Node, msg string) {
 func StripPos(root sql.Node) sql.Node {
 	zero := reflect.ValueOf(sql.Pos{})
 
-	_ = sql.Walk(sql.VisitFunc(func(node sql.Node) error {
+	_, _ = sql.Walk(sql.VisitFunc(func(node sql.Node) (sql.Node, error) {
 		value := reflect.Indirect(reflect.ValueOf(node))
 		for i := 0; i < value.NumField(); i++ {
 			if field := value.Field(i); field.Type() == zero.Type() {
 				field.Set(zero)
 			}
 		}
-		return nil
+		return node, nil
 	}), root)
 	return root
 }
diff --git a/sql2/walk.go b/sql2/walk.go
index e8592a7b6..c04ca76ba 100644
--- a/sql2/walk.go
+++ b/sql2/walk.go
@@ -18,8 +18,8 @@ package sql2
 // If the result visitor w is not nil, Walk visits each of the children
 // of node with the visitor w, followed by a call of w.Visit(nil).
 type Visitor interface {
-	Visit(node Node) (w Visitor, err error)
-	VisitEnd(node Node) error
+	Visit(node Node) (w Visitor, n Node, err error)
+	VisitEnd(node Node) (Node, error)
 }
 
 // Walk traverses an AST in depth-first order: It starts by calling
@@ -27,606 +27,774 @@ type Visitor interface {
 // v.Visit(node) is not nil, Walk is invoked recursively with visitor
 // w for each of the non-nil children of node, followed by a call of
 // w.Visit(nil).
-func Walk(v Visitor, node Node) error {
+func Walk(v Visitor, node Node) (Node, error) {
 	return walk(v, node)
 }
 
-func walk(v Visitor, node Node) (err error) {
+func walk(v Visitor, node Node) (_ Node, err error) {
 	// Visit the node itself
-	if v, err = v.Visit(node); err != nil {
-		return err
+	if v, node, err = v.Visit(node); err != nil {
+		return node, err
 	} else if v == nil {
-		return nil
+		return node, nil
 	}
 
 	// Visit node's children.
 	switch n := node.(type) {
 	case *Assignment:
 		if err := walkIdentList(v, n.Columns); err != nil {
-			return err
+			return node, err
 		}
-		if err := walkExpr(v, n.Expr); err != nil {
-			return err
+		if err := walkExpr(v, &n.Expr); err != nil {
+			return node, err
 		}
 
 	case *ExplainStatement:
 		if n.Stmt != nil {
-			if err := walk(v, n.Stmt); err != nil {
-				return err
+			if stmt, err := walk(v, n.Stmt); err != nil {
+				return node, err
+			} else if stmt == nil {
+				n.Stmt = nil
+			} else {
+				n.Stmt = stmt.(Statement)
 			}
 		}
 
 	case *RollbackStatement:
-		if err := walkIdent(v, n.SavepointName); err != nil {
-			return err
+		if err := walkIdent(v, &n.SavepointName); err != nil {
+			return node, err
 		}
 
 	case *SavepointStatement:
-		if err := walkIdent(v, n.Name); err != nil {
-			return err
+		if err := walkIdent(v, &n.Name); err != nil {
+			return node, err
 		}
 
 	case *ReleaseStatement:
-		if err := walkIdent(v, n.Name); err != nil {
-			return err
+		if err := walkIdent(v, &n.Name); err != nil {
+			return node, err
 		}
 
 	case *CreateTableStatement:
-		if err := walkIdent(v, n.Name); err != nil {
-			return err
+		if err := walkIdent(v, &n.Name); err != nil {
+			return node, err
 		}
 		if err := walkColumnDefinitionList(v, n.Columns); err != nil {
-			return err
+			return node, err
 		}
 		if err := walkConstraintList(v, n.Constraints); err != nil {
-			return err
+			return node, err
 		}
 		if n.Select != nil {
-			if err := walk(v, n.Select); err != nil {
-				return err
+			if sel, err := walk(v, n.Select); err != nil {
+				return node, err
+			} else if sel != nil {
+				n.Select = sel.(*SelectStatement)
+			} else {
+				n.Select = nil
 			}
 		}
 
 	case *AlterTableStatement:
-		if err := walkIdent(v, n.Name); err != nil {
-			return err
+		if err := walkIdent(v, &n.Name); err != nil {
+			return node, err
 		}
-		if err := walkIdent(v, n.NewName); err != nil {
-			return err
+		if err := walkIdent(v, &n.NewName); err != nil {
+			return node, err
 		}
-		if err := walkIdent(v, n.ColumnName); err != nil {
-			return err
+		if err := walkIdent(v, &n.ColumnName); err != nil {
+			return node, err
 		}
-		if err := walkIdent(v, n.NewColumnName); err != nil {
-			return err
+		if err := walkIdent(v, &n.NewColumnName); err != nil {
+			return node, err
 		}
 		if n.ColumnDef != nil {
-			if err := walk(v, n.ColumnDef); err != nil {
-				return err
+			if def, err := walk(v, n.ColumnDef); err != nil {
+				return node, err
+			} else if def != nil {
+				n.ColumnDef = def.(*ColumnDefinition)
+			} else {
+				n.ColumnDef = nil
 			}
 		}
 
 	case *AnalyzeStatement:
-		if err := walkIdent(v, n.Name); err != nil {
-			return err
+		if err := walkIdent(v, &n.Name); err != nil {
+			return node, err
 		}
 
 	case *CreateViewStatement:
-		if err := walkIdent(v, n.Name); err != nil {
-			return err
+		if err := walkIdent(v, &n.Name); err != nil {
+			return node, err
 		}
 		if err := walkIdentList(v, n.Columns); err != nil {
-			return err
+			return node, err
 		}
 		if n.Select != nil {
-			if err := walk(v, n.Select); err != nil {
-				return err
+			if sel, err := walk(v, n.Select); err != nil {
+				return node, err
+			} else if sel != nil {
+				n.Select = sel.(*SelectStatement)
+			} else {
+				n.Select = nil
 			}
 		}
 
 	case *DropTableStatement:
-		if err := walkIdent(v, n.Name); err != nil {
-			return err
+		if err := walkIdent(v, &n.Name); err != nil {
+			return node, err
 		}
 
 	case *DropViewStatement:
-		if err := walkIdent(v, n.Name); err != nil {
-			return err
+		if err := walkIdent(v, &n.Name); err != nil {
+			return node, err
 		}
 
 	case *DropIndexStatement:
-		if err := walkIdent(v, n.Name); err != nil {
-			return err
+		if err := walkIdent(v, &n.Name); err != nil {
+			return node, err
 		}
 
 	case *DropTriggerStatement:
-		if err := walkIdent(v, n.Name); err != nil {
-			return err
+		if err := walkIdent(v, &n.Name); err != nil {
+			return node, err
 		}
 
 	case *CreateIndexStatement:
-		if err := walkIdent(v, n.Name); err != nil {
-			return err
+		if err := walkIdent(v, &n.Name); err != nil {
+			return node, err
 		}
-		if err := walkIdent(v, n.Table); err != nil {
-			return err
+		if err := walkIdent(v, &n.Table); err != nil {
+			return node, err
 		}
 		if err := walkIndexedColumnList(v, n.Columns); err != nil {
-			return err
+			return node, err
 		}
-		if err := walkExpr(v, n.WhereExpr); err != nil {
-			return err
+		if err := walkExpr(v, &n.WhereExpr); err != nil {
+			return node, err
 		}
 
 	case *CreateTriggerStatement:
-		if err := walkIdent(v, n.Name); err != nil {
-			return err
+		if err := walkIdent(v, &n.Name); err != nil {
+			return node, err
 		}
 		if err := walkIdentList(v, n.UpdateOfColumns); err != nil {
-			return err
+			return node, err
 		}
-		if err := walkIdent(v, n.Table); err != nil {
-			return err
+		if err := walkIdent(v, &n.Table); err != nil {
+			return node, err
 		}
-		if err := walkExpr(v, n.WhenExpr); err != nil {
-			return err
+		if err := walkExpr(v, &n.WhenExpr); err != nil {
+			return node, err
 		}
-		for _, x := range n.Body {
-			if err := walk(v, x); err != nil {
-				return err
+		for i := range n.Body {
+			if body, err := walk(v, n.Body[i]); err != nil {
+				return node, err
+			} else if body != nil {
+				n.Body[i] = body.(Statement)
+			} else {
+				n.Body[i] = nil
 			}
 		}
 
 	case *SelectStatement:
 		if n.WithClause != nil {
-			if err := walk(v, n.WithClause); err != nil {
-				return err
+			if clause, err := walk(v, n.WithClause); err != nil {
+				return node, err
+			} else if clause != nil {
+				n.WithClause = clause.(*WithClause)
+			} else {
+				n.WithClause = nil
 			}
 		}
-		for _, x := range n.ValueLists {
-			if err := walk(v, x); err != nil {
-				return err
+		for i := range n.ValueLists {
+			if list, err := walk(v, n.ValueLists[i]); err != nil {
+				return node, err
+			} else if list != nil {
+				n.ValueLists[i] = list.(*ExprList)
+			} else {
+				n.ValueLists[i] = nil
 			}
 		}
-		for _, x := range n.Columns {
-			if err := walk(v, x); err != nil {
-				return err
+		for i := range n.Columns {
+			if col, err := walk(v, n.Columns[i]); err != nil {
+				return node, err
+			} else if col != nil {
+				n.Columns[i] = col.(*ResultColumn)
+			} else {
+				n.Columns[i] = nil
 			}
 		}
 		if n.Source != nil {
-			if err := walk(v, n.Source); err != nil {
-				return err
+			if src, err := walk(v, n.Source); err != nil {
+				return node, err
+			} else if src != nil {
+				n.Source = n.Source.(Source)
+			} else {
+				n.Source = nil
 			}
 		}
-		if err := walkExpr(v, n.WhereExpr); err != nil {
-			return err
+		if err := walkExpr(v, &n.WhereExpr); err != nil {
+			return node, err
 		}
 		if err := walkExprList(v, n.GroupByExprs); err != nil {
-			return err
+			return node, err
 		}
-		if err := walkExpr(v, n.HavingExpr); err != nil {
-			return err
+		if err := walkExpr(v, &n.HavingExpr); err != nil {
+			return node, err
 		}
-		for _, x := range n.Windows {
-			if err := walk(v, x); err != nil {
-				return err
+		for i := range n.Windows {
+			if w, err := walk(v, n.Windows[i]); err != nil {
+				return node, err
+			} else if w != nil {
+				n.Windows[i] = w.(*Window)
+			} else {
+				n.Windows[i] = nil
 			}
 		}
 		if n.Compound != nil {
-			if err := walk(v, n.Compound); err != nil {
-				return err
+			if stmt, err := walk(v, n.Compound); err != nil {
+				return node, err
+			} else if stmt != nil {
+				n.Compound = stmt.(*SelectStatement)
+			} else {
+				n.Compound = nil
 			}
 		}
-		for _, x := range n.OrderingTerms {
-			if err := walk(v, x); err != nil {
-				return err
+		for i := range n.OrderingTerms {
+			if term, err := walk(v, n.OrderingTerms[i]); err != nil {
+				return node, err
+			} else if term != nil {
+				n.OrderingTerms[i] = term.(*OrderingTerm)
+			} else {
+				n.OrderingTerms[i] = nil
 			}
 		}
-		if err := walkExpr(v, n.LimitExpr); err != nil {
-			return err
+		if err := walkExpr(v, &n.LimitExpr); err != nil {
+			return node, err
 		}
-		if err := walkExpr(v, n.OffsetExpr); err != nil {
-			return err
+		if err := walkExpr(v, &n.OffsetExpr); err != nil {
+			return node, err
 		}
 
 	case *InsertStatement:
 		if n.WithClause != nil {
-			if err := walk(v, n.WithClause); err != nil {
-				return err
+			if clause, err := walk(v, n.WithClause); err != nil {
+				return node, err
+			} else if clause != nil {
+				n.WithClause = clause.(*WithClause)
+			} else {
+				n.WithClause = nil
 			}
 		}
-		if err := walkIdent(v, n.Table); err != nil {
-			return err
+		if err := walkIdent(v, &n.Table); err != nil {
+			return node, err
 		}
-		if err := walkIdent(v, n.Alias); err != nil {
-			return err
+		if err := walkIdent(v, &n.Alias); err != nil {
+			return node, err
 		}
 		if err := walkIdentList(v, n.Columns); err != nil {
-			return err
+			return node, err
 		}
-		for _, x := range n.ValueLists {
-			if err := walk(v, x); err != nil {
-				return err
+		for i := range n.ValueLists {
+			if list, err := walk(v, n.ValueLists[i]); err != nil {
+				return node, err
+			} else if list != nil {
+				n.ValueLists[i] = list.(*ExprList)
+			} else {
+				n.ValueLists[i] = nil
 			}
 		}
 		if n.Select != nil {
-			if err := walk(v, n.Select); err != nil {
-				return err
+			if sel, err := walk(v, n.Select); err != nil {
+				return node, err
+			} else if sel != nil {
+				n.Select = sel.(*SelectStatement)
+			} else {
+				n.Select = nil
 			}
 		}
 		if n.UpsertClause != nil {
-			if err := walk(v, n.UpsertClause); err != nil {
-				return err
+			if clause, err := walk(v, n.UpsertClause); err != nil {
+				return node, err
+			} else if clause != nil {
+				n.UpsertClause = clause.(*UpsertClause)
+			} else {
+				n.UpsertClause = nil
 			}
 		}
 
 	case *UpdateStatement:
 		if n.WithClause != nil {
-			if err := walk(v, n.WithClause); err != nil {
-				return err
+			if clause, err := walk(v, n.WithClause); err != nil {
+				return node, err
+			} else if clause != nil {
+				n.WithClause = clause.(*WithClause)
+			} else {
+				n.WithClause = nil
 			}
 		}
 		if n.Table != nil {
-			if err := walk(v, n.Table); err != nil {
-				return err
+			if tbl, err := walk(v, n.Table); err != nil {
+				return node, err
+			} else if tbl != nil {
+				n.Table = tbl.(*QualifiedTableName)
+			} else {
+				n.Table = nil
 			}
 		}
-		for _, x := range n.Assignments {
-			if err := walk(v, x); err != nil {
-				return err
+		for i := range n.Assignments {
+			if assign, err := walk(v, n.Assignments[i]); err != nil {
+				return node, err
+			} else if assign != nil {
+				n.Assignments[i] = assign.(*Assignment)
+			} else {
+				n.Assignments[i] = nil
 			}
 		}
-		if err := walkExpr(v, n.WhereExpr); err != nil {
-			return err
+		if err := walkExpr(v, &n.WhereExpr); err != nil {
+			return node, err
 		}
 
 	case *UpsertClause:
 		if err := walkIndexedColumnList(v, n.Columns); err != nil {
-			return err
+			return node, err
 		}
-		if err := walkExpr(v, n.WhereExpr); err != nil {
-			return err
+		if err := walkExpr(v, &n.WhereExpr); err != nil {
+			return node, err
 		}
-		for _, x := range n.Assignments {
-			if err := walk(v, x); err != nil {
-				return err
+		for i := range n.Assignments {
+			if assign, err := walk(v, n.Assignments[i]); err != nil {
+				return node, err
+			} else if assign != nil {
+				n.Assignments[i] = assign.(*Assignment)
+			} else {
+				n.Assignments[i] = nil
 			}
 		}
-		if err := walkExpr(v, n.UpdateWhereExpr); err != nil {
-			return err
+		if err := walkExpr(v, &n.UpdateWhereExpr); err != nil {
+			return node, err
 		}
 
 	case *DeleteStatement:
 		if n.WithClause != nil {
-			if err := walk(v, n.WithClause); err != nil {
-				return err
+			if clause, err := walk(v, n.WithClause); err != nil {
+				return node, err
+			} else if clause != nil {
+				n.WithClause = clause.(*WithClause)
+			} else {
+				n.WithClause = nil
 			}
 		}
 		if n.Table != nil {
-			if err := walk(v, n.Table); err != nil {
-				return err
+			if tbl, err := walk(v, n.Table); err != nil {
+				return node, err
+			} else if tbl != nil {
+				n.Table = tbl.(*QualifiedTableName)
+			} else {
+				n.Table = nil
 			}
 		}
-		if err := walkExpr(v, n.WhereExpr); err != nil {
-			return err
+		if err := walkExpr(v, &n.WhereExpr); err != nil {
+			return node, err
 		}
-		for _, x := range n.OrderingTerms {
-			if err := walk(v, x); err != nil {
-				return err
+		for i := range n.OrderingTerms {
+			if term, err := walk(v, n.OrderingTerms[i]); err != nil {
+				return node, err
+			} else if term != nil {
+				n.OrderingTerms[i] = term.(*OrderingTerm)
+			} else {
+				n.OrderingTerms[i] = nil
 			}
 		}
-		if err := walkExpr(v, n.LimitExpr); err != nil {
-			return err
+		if err := walkExpr(v, &n.LimitExpr); err != nil {
+			return node, err
 		}
-		if err := walkExpr(v, n.OffsetExpr); err != nil {
-			return err
+		if err := walkExpr(v, &n.OffsetExpr); err != nil {
+			return node, err
 		}
 
 	case *PrimaryKeyConstraint:
-		if err := walkIdent(v, n.Name); err != nil {
-			return err
+		if err := walkIdent(v, &n.Name); err != nil {
+			return node, err
 		}
 		if err := walkIdentList(v, n.Columns); err != nil {
-			return err
+			return node, err
 		}
 
 	case *NotNullConstraint:
-		if err := walkIdent(v, n.Name); err != nil {
-			return err
+		if err := walkIdent(v, &n.Name); err != nil {
+			return node, err
 		}
 
 	case *UniqueConstraint:
-		if err := walkIdent(v, n.Name); err != nil {
-			return err
+		if err := walkIdent(v, &n.Name); err != nil {
+			return node, err
 		}
 		if err := walkIdentList(v, n.Columns); err != nil {
-			return err
+			return node, err
 		}
 
 	case *CheckConstraint:
-		if err := walkIdent(v, n.Name); err != nil {
-			return err
+		if err := walkIdent(v, &n.Name); err != nil {
+			return node, err
 		}
-		if err := walkExpr(v, n.Expr); err != nil {
-			return err
+		if err := walkExpr(v, &n.Expr); err != nil {
+			return node, err
 		}
 
 	case *DefaultConstraint:
-		if err := walkIdent(v, n.Name); err != nil {
-			return err
+		if err := walkIdent(v, &n.Name); err != nil {
+			return node, err
 		}
-		if err := walkExpr(v, n.Expr); err != nil {
-			return err
+		if err := walkExpr(v, &n.Expr); err != nil {
+			return node, err
 		}
 
 	case *ForeignKeyConstraint:
-		if err := walkIdent(v, n.Name); err != nil {
-			return err
+		if err := walkIdent(v, &n.Name); err != nil {
+			return node, err
 		}
 		if err := walkIdentList(v, n.Columns); err != nil {
-			return err
+			return node, err
 		}
-		if err := walkIdent(v, n.ForeignTable); err != nil {
-			return err
+		if err := walkIdent(v, &n.ForeignTable); err != nil {
+			return node, err
 		}
 		if err := walkIdentList(v, n.ForeignColumns); err != nil {
-			return err
+			return node, err
 		}
-		for _, x := range n.Args {
-			if err := walk(v, x); err != nil {
-				return err
+		for i := range n.Args {
+			if arg, err := walk(v, n.Args[i]); err != nil {
+				return node, err
+			} else if arg != nil {
+				n.Args[i] = arg.(*ForeignKeyArg)
+			} else {
+				n.Args[i] = nil
 			}
 		}
 
 	case *ParenExpr:
-		if err := walkExpr(v, n.X); err != nil {
-			return err
+		if err := walkExpr(v, &n.X); err != nil {
+			return node, err
 		}
 
 	case *UnaryExpr:
-		if err := walkExpr(v, n.X); err != nil {
-			return err
+		if err := walkExpr(v, &n.X); err != nil {
+			return node, err
 		}
 
 	case *BinaryExpr:
-		if err := walkExpr(v, n.X); err != nil {
-			return err
+		if err := walkExpr(v, &n.X); err != nil {
+			return node, err
 		}
-		if err := walkExpr(v, n.Y); err != nil {
-			return err
+		if err := walkExpr(v, &n.Y); err != nil {
+			return node, err
 		}
 
 	case *CastExpr:
-		if err := walkExpr(v, n.X); err != nil {
-			return err
+		if err := walkExpr(v, &n.X); err != nil {
+			return node, err
 		}
 		if n.Type != nil {
-			if err := walk(v, n.Type); err != nil {
-				return err
+			if typ, err := walk(v, n.Type); err != nil {
+				return node, err
+			} else if typ != nil {
+				n.Type = typ.(*Type)
+			} else {
+				n.Type = nil
 			}
 		}
 
 	case *CaseBlock:
-		if err := walkExpr(v, n.Condition); err != nil {
-			return err
+		if err := walkExpr(v, &n.Condition); err != nil {
+			return node, err
 		}
-		if err := walkExpr(v, n.Body); err != nil {
-			return err
+		if err := walkExpr(v, &n.Body); err != nil {
+			return node, err
 		}
 
 	case *CaseExpr:
-		if err := walkExpr(v, n.Operand); err != nil {
-			return err
+		if err := walkExpr(v, &n.Operand); err != nil {
+			return node, err
 		}
-		for _, x := range n.Blocks {
-			if err := walk(v, x); err != nil {
-				return err
+		for i := range n.Blocks {
+			if blk, err := walk(v, n.Blocks[i]); err != nil {
+				return node, err
+			} else if blk != nil {
+				n.Blocks[i] = blk.(*CaseBlock)
+			} else {
+				n.Blocks[i] = nil
 			}
 		}
-		if err := walkExpr(v, n.ElseExpr); err != nil {
-			return err
+		if err := walkExpr(v, &n.ElseExpr); err != nil {
+			return node, err
 		}
 
 	case *ExprList:
 		if err := walkExprList(v, n.Exprs); err != nil {
-			return err
+			return node, err
 		}
 
 	case *QualifiedRef:
-		if err := walkIdent(v, n.Table); err != nil {
-			return err
+		if err := walkIdent(v, &n.Table); err != nil {
+			return node, err
 		}
-		if err := walkIdent(v, n.Column); err != nil {
-			return err
+		if err := walkIdent(v, &n.Column); err != nil {
+			return node, err
 		}
 
 	case *Call:
-		if err := walkIdent(v, n.Name); err != nil {
-			return err
+		if err := walkIdent(v, &n.Name); err != nil {
+			return node, err
 		}
 		if err := walkExprList(v, n.Args); err != nil {
-			return err
+			return node, err
 		}
 		if n.Filter != nil {
-			if err := walk(v, n.Filter); err != nil {
-				return err
+			if filter, err := walk(v, n.Filter); err != nil {
+				return node, err
+			} else if filter != nil {
+				n.Filter = filter.(*FilterClause)
+			} else {
+				n.Filter = nil
 			}
 		}
 		if n.Over != nil {
-			if err := walk(v, n.Over); err != nil {
-				return err
+			if over, err := walk(v, n.Over); err != nil {
+				return node, err
+			} else if over != nil {
+				n.Over = over.(*OverClause)
+			} else {
+				n.Over = nil
 			}
 		}
 
 	case *FilterClause:
-		if err := walkExpr(v, n.X); err != nil {
-			return err
+		if err := walkExpr(v, &n.X); err != nil {
+			return node, err
 		}
 
 	case *OverClause:
-		if err := walkIdent(v, n.Name); err != nil {
-			return err
+		if err := walkIdent(v, &n.Name); err != nil {
+			return node, err
 		}
 		if n.Definition != nil {
-			if err := walk(v, n.Definition); err != nil {
-				return err
+			if def, err := walk(v, n.Definition); err != nil {
+				return node, err
+			} else if def != nil {
+				n.Definition = def.(*WindowDefinition)
+			} else {
+				n.Definition = nil
 			}
 		}
 
 	case *OrderingTerm:
-		if err := walkExpr(v, n.X); err != nil {
-			return err
+		if err := walkExpr(v, &n.X); err != nil {
+			return node, err
 		}
 
 	case *FrameSpec:
-		if err := walkExpr(v, n.X); err != nil {
-			return err
+		if err := walkExpr(v, &n.X); err != nil {
+			return node, err
 		}
-		if err := walkExpr(v, n.Y); err != nil {
-			return err
+		if err := walkExpr(v, &n.Y); err != nil {
+			return node, err
 		}
 
 	case *Range:
-		if err := walkExpr(v, n.X); err != nil {
-			return err
+		if err := walkExpr(v, &n.X); err != nil {
+			return node, err
 		}
-		if err := walkExpr(v, n.Y); err != nil {
-			return err
+		if err := walkExpr(v, &n.Y); err != nil {
+			return node, err
 		}
 
 	case *Raise:
 		if n.Error != nil {
-			if err := walk(v, n.Error); err != nil {
-				return err
+			if e, err := walk(v, n.Error); err != nil {
+				return node, err
+			} else if e != nil {
+				n.Error = e.(*StringLit)
+			} else {
+				n.Error = nil
 			}
 		}
 
 	case *Exists:
 		if n.Select != nil {
-			if err := walk(v, n.Select); err != nil {
-				return err
+			if sel, err := walk(v, n.Select); err != nil {
+				return node, err
+			} else if sel != nil {
+				n.Select = sel.(*SelectStatement)
+			} else {
+				n.Select = nil
 			}
 		}
 
 	case *ParenSource:
 		if n.X != nil {
-			if err := walk(v, n.X); err != nil {
-				return err
+			if x, err := walk(v, n.X); err != nil {
+				return node, err
+			} else if x != nil {
+				n.X = x.(Source)
+			} else {
+				n.X = nil
 			}
 		}
-		if err := walkIdent(v, n.Alias); err != nil {
-			return err
+		if err := walkIdent(v, &n.Alias); err != nil {
+			return node, err
 		}
 
 	case *QualifiedTableName:
-		if err := walkIdent(v, n.Name); err != nil {
-			return err
+		if err := walkIdent(v, &n.Name); err != nil {
+			return node, err
 		}
-		if err := walkIdent(v, n.Alias); err != nil {
-			return err
+		if err := walkIdent(v, &n.Alias); err != nil {
+			return node, err
 		}
-		if err := walkIdent(v, n.Index); err != nil {
-			return err
+		if err := walkIdent(v, &n.Index); err != nil {
+			return node, err
 		}
 
 	case *JoinClause:
 		if n.X != nil {
-			if err := walk(v, n.X); err != nil {
-				return err
+			if x, err := walk(v, n.X); err != nil {
+				return node, err
+			} else if x != nil {
+				n.X = x.(Source)
+			} else {
+				n.X = nil
 			}
 		}
 		if n.Operator != nil {
-			if err := walk(v, n.Operator); err != nil {
-				return err
+			if op, err := walk(v, n.Operator); err != nil {
+				return node, err
+			} else if op != nil {
+				n.Operator = op.(*JoinOperator)
+			} else {
+				n.Operator = nil
 			}
 		}
 		if n.Y != nil {
-			if err := walk(v, n.Y); err != nil {
-				return err
+			if y, err := walk(v, n.Y); err != nil {
+				return node, err
+			} else if y != nil {
+				n.Y = y.(Source)
+			} else {
+				n.Y = nil
 			}
 		}
 		if n.Constraint != nil {
-			if err := walk(v, n.Constraint); err != nil {
-				return err
+			if cons, err := walk(v, n.Constraint); err != nil {
+				return node, err
+			} else if cons != nil {
+				n.Constraint = cons.(JoinConstraint)
+			} else {
+				n.Constraint = nil
 			}
 		}
 
 	case *OnConstraint:
-		if err := walkExpr(v, n.X); err != nil {
-			return err
+		if err := walkExpr(v, &n.X); err != nil {
+			return node, err
 		}
 
 	case *UsingConstraint:
 		if err := walkIdentList(v, n.Columns); err != nil {
-			return err
+			return node, err
 		}
 
 	case *ColumnDefinition:
-		if err := walkIdent(v, n.Name); err != nil {
-			return err
+		if err := walkIdent(v, &n.Name); err != nil {
+			return node, err
 		}
 		if n.Type != nil {
-			if err := walk(v, n.Type); err != nil {
-				return err
+			if typ, err := walk(v, n.Type); err != nil {
+				return node, err
+			} else if typ != nil {
+				n.Type = typ.(*Type)
+			} else {
+				n.Type = nil
 			}
 		}
 		if err := walkConstraintList(v, n.Constraints); err != nil {
-			return err
+			return node, err
 		}
 
 	case *ResultColumn:
-		if err := walkExpr(v, n.Expr); err != nil {
-			return err
+		if err := walkExpr(v, &n.Expr); err != nil {
+			return node, err
 		}
-		if err := walkIdent(v, n.Alias); err != nil {
-			return err
+		if err := walkIdent(v, &n.Alias); err != nil {
+			return node, err
 		}
 
 	case *IndexedColumn:
-		if err := walkExpr(v, n.X); err != nil {
-			return err
+		if err := walkExpr(v, &n.X); err != nil {
+			return node, err
 		}
 
 	case *Window:
-		if err := walkIdent(v, n.Name); err != nil {
-			return err
+		if err := walkIdent(v, &n.Name); err != nil {
+			return node, err
 		}
 		if n.Definition != nil {
-			if err := walk(v, n.Definition); err != nil {
-				return err
+			if def, err := walk(v, n.Definition); err != nil {
+				return node, err
+			} else if def != nil {
+				n.Definition = def.(*WindowDefinition)
+			} else {
+				n.Definition = nil
 			}
 		}
 
 	case *WindowDefinition:
-		if err := walkIdent(v, n.Base); err != nil {
-			return err
+		if err := walkIdent(v, &n.Base); err != nil {
+			return node, err
 		}
 		if err := walkExprList(v, n.Partitions); err != nil {
-			return err
+			return node, err
 		}
-		for _, x := range n.OrderingTerms {
-			if err := walk(v, x); err != nil {
-				return err
+		for i := range n.OrderingTerms {
+			if term, err := walk(v, n.OrderingTerms[i]); err != nil {
+				return node, err
+			} else if term != nil {
+				n.OrderingTerms[i] = term.(*OrderingTerm)
+			} else {
+				n.OrderingTerms[i] = nil
 			}
 		}
 		if n.Frame != nil {
-			if err := walk(v, n.Frame); err != nil {
-				return err
+			if frame, err := walk(v, n.Frame); err != nil {
+				return node, err
+			} else if frame != nil {
+				n.Frame = frame.(*FrameSpec)
+			} else {
+				n.Frame = nil
 			}
 		}
 
 	case *Type:
-		if err := walkIdent(v, n.Name); err != nil {
-			return err
+		if err := walkIdent(v, &n.Name); err != nil {
+			return node, err
 		}
 		if n.Precision != nil {
-			if err := walk(v, n.Precision); err != nil {
-				return err
+			if p, err := walk(v, n.Precision); err != nil {
+				return node, err
+			} else if p != nil {
+				n.Precision = p.(*NumberLit)
+			} else {
+				n.Precision = nil
 			}
 		}
 		if n.Scale != nil {
-			if err := walk(v, n.Scale); err != nil {
-				return err
+			if scale, err := walk(v, n.Scale); err != nil {
+				return node, err
+			} else if scale != nil {
+				n.Scale = scale.(*NumberLit)
+			} else {
+				n.Scale = nil
 			}
 		}
 	}
@@ -637,59 +805,72 @@ func walk(v Visitor, node Node) (err error) {
 
 // VisitFunc represents a function type that implements Visitor.
 // Only executes on node entry.
-type VisitFunc func(Node) error
+type VisitFunc func(Node) (Node, error)
 
 // Visit executes fn. Walk visits node children if fn returns true.
-func (fn VisitFunc) Visit(node Node) (Visitor, error) {
-	if err := fn(node); err != nil {
-		return nil, err
+func (fn VisitFunc) Visit(node Node) (Visitor, Node, error) {
+	node, err := fn(node)
+	if err != nil {
+		return nil, nil, err
 	}
-	return fn, nil
+	return fn, node, nil
 }
 
 // VisitEnd is a no-op.
-func (fn VisitFunc) VisitEnd(node Node) error { return nil }
+func (fn VisitFunc) VisitEnd(node Node) (Node, error) { return node, nil }
 
 // VisitEndFunc represents a function type that implements Visitor.
 // Only executes on node exit.
-type VisitEndFunc func(Node) error
+type VisitEndFunc func(Node) (Node, error)
 
 // Visit is a no-op.
-func (fn VisitEndFunc) Visit(node Node) (Visitor, error) { return fn, nil }
+func (fn VisitEndFunc) Visit(node Node) (Visitor, Node, error) { return fn, node, nil }
 
 // VisitEnd executes fn.
-func (fn VisitEndFunc) VisitEnd(node Node) error { return fn(node) }
+func (fn VisitEndFunc) VisitEnd(node Node) (Node, error) { return fn(node) }
 
-func walkIdent(v Visitor, x *Ident) error {
-	if x != nil {
-		if err := walk(v, x); err != nil {
-			return err
-		}
+func walkIdent(v Visitor, x **Ident) error {
+	if *x == nil {
+		return nil
+	}
+
+	ident, err := walk(v, *x)
+	if err != nil {
+		return err
+	} else if ident != nil {
+		*x = ident.(*Ident)
+	} else {
+		*x = nil
 	}
 	return nil
 }
 
 func walkIdentList(v Visitor, a []*Ident) error {
-	for _, x := range a {
-		if err := walk(v, x); err != nil {
+	for i := range a {
+		if err := walkIdent(v, &a[i]); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func walkExpr(v Visitor, x Expr) error {
-	if x != nil {
-		if err := walk(v, x); err != nil {
-			return err
-		}
+func walkExpr(v Visitor, x *Expr) error {
+	if *x == nil {
+		return nil
+	}
+	if other, err := walk(v, *x); err != nil {
+		return err
+	} else if other != nil {
+		*x = other.(Expr)
+	} else {
+		*x = nil
 	}
 	return nil
 }
 
 func walkExprList(v Visitor, a []Expr) error {
-	for _, x := range a {
-		if err := walk(v, x); err != nil {
+	for i := range a {
+		if err := walkExpr(v, &a[i]); err != nil {
 			return err
 		}
 	}
@@ -697,27 +878,39 @@ func walkExprList(v Visitor, a []Expr) error {
 }
 
 func walkConstraintList(v Visitor, a []Constraint) error {
-	for _, x := range a {
-		if err := walk(v, x); err != nil {
+	for i := range a {
+		if cons, err := walk(v, a[i]); err != nil {
 			return err
+		} else if cons != nil {
+			a[i] = cons.(Constraint)
+		} else {
+			a[i] = nil
 		}
 	}
 	return nil
 }
 
 func walkIndexedColumnList(v Visitor, a []*IndexedColumn) error {
-	for _, x := range a {
-		if err := walk(v, x); err != nil {
+	for i := range a {
+		if col, err := walk(v, a[i]); err != nil {
 			return err
+		} else if col != nil {
+			a[i] = col.(*IndexedColumn)
+		} else {
+			a[i] = nil
 		}
 	}
 	return nil
 }
 
 func walkColumnDefinitionList(v Visitor, a []*ColumnDefinition) error {
-	for _, x := range a {
-		if err := walk(v, x); err != nil {
+	for i := range a {
+		if def, err := walk(v, a[i]); err != nil {
 			return err
+		} else if def != nil {
+			a[i] = def.(*ColumnDefinition)
+		} else {
+			a[i] = nil
 		}
 	}
 	return nil

From 1527eea14b20235d2935e1e05f41a952252dabf3 Mon Sep 17 00:00:00 2001
From: Ben Johnson <benbjohnson@yahoo.com>
Date: Sat, 25 Sep 2021 09:36:11 -0600
Subject: [PATCH 23/66] Fix SQL column mappings

---
 planner.go      | 151 ++++++++++++++++++++++++++++++++----------------
 planner_test.go |  58 +++++++++++++++++--
 2 files changed, 154 insertions(+), 55 deletions(-)

diff --git a/planner.go b/planner.go
index 9fd141fc8..ce89deebe 100644
--- a/planner.go
+++ b/planner.go
@@ -77,19 +77,21 @@ func (p *Planner) planAggregateSelectStatement(ctx context.Context, stmt *sql2.S
 	// TODO: Recursively traverse all expression trees.
 	var calls []*sql2.Call
 	var columns []*StmtColumn
+	var resultCols []string
 	for _, c := range stmt.Columns {
 		columns = append(columns, &StmtColumn{
 			Name: c.Name(),
 			Type: sql2.ExprDataType(c.Expr),
 		})
 
-		switch c := c.Expr.(type) {
+		switch expr := c.Expr.(type) {
 		case *sql2.Call:
-			calls = append(calls, c)
+			calls = append(calls, expr)
+			resultCols = append(resultCols, "_aggregate")
 		case *sql2.QualifiedRef:
-			// allowed
+			resultCols = append(resultCols, expr.Column.Name)
 		default:
-			return nil, fmt.Errorf("unsupported expression type in aggregate query: %T", c)
+			return nil, fmt.Errorf("unsupported expression type in aggregate query: %T", expr)
 		}
 	}
 
@@ -99,11 +101,11 @@ func (p *Planner) planAggregateSelectStatement(ctx context.Context, stmt *sql2.S
 	}
 
 	// Extract column names in GROUP BY clause.
-	var groupByColNames []string
+	var groupByCols []string
 	for _, expr := range stmt.GroupByExprs {
 		switch expr := expr.(type) {
 		case *sql2.QualifiedRef:
-			groupByColNames = append(groupByColNames, expr.Column.Name)
+			groupByCols = append(groupByCols, expr.Column.Name)
 		default:
 			return nil, fmt.Errorf("unsupported expression type in GROUP BY clause: %T", expr)
 		}
@@ -113,7 +115,7 @@ func (p *Planner) planAggregateSelectStatement(ctx context.Context, stmt *sql2.S
 	callName := strings.ToUpper(sql2.IdentName(calls[0].Name))
 	switch callName {
 	case "COUNT":
-		if len(groupByColNames) == 0 {
+		if len(groupByCols) == 0 {
 			return NewCountNode(p.executor, indexName, columns[0], cond), nil
 		}
 
@@ -136,7 +138,7 @@ func (p *Planner) planAggregateSelectStatement(ctx context.Context, stmt *sql2.S
 			}
 		}
 
-		return NewGroupByNode(p.executor, indexName, groupByColNames, columns, aggregate, cond), nil
+		return NewGroupByNode(p.executor, indexName, resultCols, groupByCols, columns, aggregate, cond), nil
 
 	case "SUM":
 		if len(calls[0].Args) != 1 {
@@ -152,7 +154,7 @@ func (p *Planner) planAggregateSelectStatement(ctx context.Context, stmt *sql2.S
 			Args: map[string]interface{}{"field": ref.Column.Name},
 		}
 
-		return NewGroupByNode(p.executor, indexName, groupByColNames, columns, aggregate, cond), nil
+		return NewGroupByNode(p.executor, indexName, resultCols, groupByCols, columns, aggregate, cond), nil
 
 	default:
 		return nil, fmt.Errorf("unsupported call in aggregate query: %s", callName)
@@ -770,6 +772,7 @@ type ExtractNode struct {
 	indexName string
 	srcs      []string
 	columns   []*StmtColumn
+	mapping   []int // map of output column indices to source column indices
 	cond      *pql.Call
 
 	result []ExtractedTableColumn
@@ -781,18 +784,26 @@ func NewExtractNode(executor *executor, indexName string, srcs []string, columns
 		cond = &pql.Call{Name: "All"}
 	}
 
-	// Ensure ID column is always the first column.
-	// TODO(benbjohnson): Don't require id first.
-	if len(srcs) > 0 && srcs[0] != "_id" {
-		srcs = append([]string{"_id"}, srcs...)
-		columns = append([]*StmtColumn{{Name: "_id", Type: sql2.DataTypeInt}}, columns...)
+	// Determine mapping between result elements & columns.
+	// We'll exclude "_id" from the source columns here as well.
+	mapping := make([]int, len(columns))
+	srcs2 := make([]string, 0, len(srcs))
+	for i := range mapping {
+		if srcs[i] == "_id" {
+			mapping[i] = -1
+			continue
+		}
+
+		mapping[i] = len(srcs2)
+		srcs2 = append(srcs2, srcs[i])
 	}
 
 	return &ExtractNode{
 		executor:  executor,
 		indexName: indexName,
-		srcs:      srcs,    // source column names
+		srcs:      srcs2,   // source column names (excluding "id")
 		columns:   columns, // external column alias
+		mapping:   mapping,
 		cond:      cond,
 		row:       make([]interface{}, len(srcs)),
 	}
@@ -808,10 +819,12 @@ func (n *ExtractNode) First(ctx context.Context) error {
 }
 
 func (n *ExtractNode) Next(ctx context.Context) error {
+	// Fetch results if we haven't yet.
 	if err := n.init(ctx); err != nil {
 		return err
 	}
 
+	// Exit if no result rows remain.
 	if len(n.result) == 0 {
 		for i := range n.row {
 			n.row[i] = nil
@@ -819,18 +832,25 @@ func (n *ExtractNode) Next(ctx context.Context) error {
 		return sql.ErrNoRows
 	}
 
-	// Copy ID value to current row.
-	result := n.result[0]
-	if result.Column.Keyed {
-		n.row[0] = result.Column.Key
-	} else {
-		n.row[0] = int64(result.Column.ID)
+	// Map result elements to row elements.
+	for i, index := range n.mapping {
+		result := n.result[0]
+
+		// Map row array element to position in result row.
+		if index >= 0 {
+			n.row[i] = result.Rows[index]
+			continue
+		}
+
+		// Otherwise use ID for value.
+		if result.Column.Keyed {
+			n.row[i] = result.Column.Key
+		} else {
+			n.row[i] = int64(result.Column.ID)
+		}
 	}
 
-	// Copy values to current row.
-	for i, v := range result.Rows {
-		n.row[i+1] = v
-	}
+	// Move to next result element.
 	n.result = n.result[1:]
 
 	return nil
@@ -844,7 +864,7 @@ func (n *ExtractNode) init(ctx context.Context) error {
 	// Generate PQL query with all specified rows.
 	// Skip first column as it is the ID column.
 	call := &pql.Call{Name: "Extract", Children: []*pql.Call{n.cond}}
-	for _, src := range n.srcs[1:] {
+	for _, src := range n.srcs {
 		call.Children = append(call.Children,
 			&pql.Call{
 				Name: "Rows",
@@ -932,12 +952,13 @@ func (n *CountNode) Row() []interface{} { return n.row }
 
 // GroupByNode executes an aggregate with a GROUP BY against a FeatureBase index.
 type GroupByNode struct {
-	executor  *executor
-	indexName string
-	srcs      []string
-	columns   []*StmtColumn
-	aggregate *pql.Call
-	cond      *pql.Call
+	executor    *executor
+	indexName   string
+	groupByCols []string
+	columns     []*StmtColumn
+	mapping     []int
+	aggregate   *pql.Call
+	cond        *pql.Call
 
 	result *GroupCounts
 	index  int
@@ -945,15 +966,27 @@ type GroupByNode struct {
 	row []interface{}
 }
 
-func NewGroupByNode(executor *executor, indexName string, srcs []string, columns []*StmtColumn, aggregate, cond *pql.Call) *GroupByNode {
+func NewGroupByNode(executor *executor, indexName string, resultCols, groupByCols []string, columns []*StmtColumn, aggregate, cond *pql.Call) *GroupByNode {
+	// Map result columns to output columns.
+	mapping := make([]int, len(columns))
+	for i := range mapping {
+		if resultCols[i] == "_aggregate" {
+			mapping[i] = -1
+			continue
+		}
+
+		mapping[i] = stringSliceIndex(groupByCols, resultCols[i])
+	}
+
 	return &GroupByNode{
-		executor:  executor,
-		indexName: indexName,
-		srcs:      srcs,
-		columns:   columns,
-		aggregate: aggregate,
-		cond:      cond,
-		row:       make([]interface{}, len(srcs)+1),
+		executor:    executor,
+		indexName:   indexName,
+		groupByCols: groupByCols,
+		columns:     columns,
+		mapping:     mapping,
+		aggregate:   aggregate,
+		cond:        cond,
+		row:         make([]interface{}, len(columns)),
 	}
 }
 
@@ -983,19 +1016,25 @@ func (n *GroupByNode) Next(ctx context.Context) (err error) {
 	group := n.result.groups[n.index]
 	n.index++
 
-	if n.aggregate != nil {
-		n.row[0] = int64(group.Agg)
-	} else {
-		n.row[0] = int64(group.Count)
-	}
+	for i, index := range n.mapping {
+		// Assign aggregate to unmapped column.
+		if index == -1 {
+			if n.aggregate != nil {
+				n.row[i] = int64(group.Agg)
+			} else {
+				n.row[i] = int64(group.Count)
+			}
+			continue
+		}
 
-	for i, g := range group.Group {
+		// Otherwise map from group value to result column index.
+		g := group.Group[index]
 		if g.Value != nil {
-			n.row[i+1] = *g.Value
+			n.row[i] = *g.Value
 		} else if g.RowKey != "" {
-			n.row[i+1] = g.RowKey
+			n.row[i] = g.RowKey
 		} else {
-			n.row[i+1] = int64(g.RowID)
+			n.row[i] = int64(g.RowID)
 		}
 	}
 
@@ -1010,9 +1049,9 @@ func (n *GroupByNode) fetch(ctx context.Context) (*GroupCounts, error) {
 	}
 
 	// Choose fields to group by.
-	for _, src := range n.srcs {
+	for _, name := range n.groupByCols {
 		call.Children = append(call.Children, &pql.Call{
-			Name: "Rows", Args: map[string]interface{}{"_field": src},
+			Name: "Rows", Args: map[string]interface{}{"_field": name},
 		})
 	}
 
@@ -1059,3 +1098,13 @@ func sourceTableName(source sql2.Source) (string, error) {
 		return "", fmt.Errorf("unexpected source type: %T", source)
 	}
 }
+
+// stringSliceIndex returns position of v in a. Returns -1 if not found.
+func stringSliceIndex(a []string, v string) int {
+	for i := range a {
+		if a[i] == v {
+			return i
+		}
+	}
+	return -1
+}
diff --git a/planner_test.go b/planner_test.go
index e92bd77a6..68d7505e4 100644
--- a/planner_test.go
+++ b/planner_test.go
@@ -210,18 +210,18 @@ func TestPlanner_Select(t *testing.T) {
 	}
 
 	t.Run("UnqualifiedColumns", func(t *testing.T) {
-		results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT _id, a, b FROM i0`)
+		results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT a, b, _id FROM i0`)
 		if diff := cmp.Diff([][]interface{}{
-			{int64(1), int64(10), int64(100)},
-			{int64(2), int64(20), int64(200)},
+			{int64(10), int64(100), int64(1)},
+			{int64(20), int64(200), int64(2)},
 		}, results); diff != "" {
 			t.Fatal(diff)
 		}
 
 		if diff := cmp.Diff([]*pilosa.StmtColumn{
-			{Name: "_id", Type: "INT"},
 			{Name: "a", Type: "INT"},
 			{Name: "b", Type: "INT"},
+			{Name: "_id", Type: "INT"},
 		}, columns); diff != "" {
 			t.Fatal(diff)
 		}
@@ -281,6 +281,23 @@ func TestPlanner_Select(t *testing.T) {
 		}
 	})
 
+	t.Run("NoIdentifier", func(t *testing.T) {
+		results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT a, b FROM i0`)
+		if diff := cmp.Diff([][]interface{}{
+			{int64(10), int64(100)},
+			{int64(20), int64(200)},
+		}, results); diff != "" {
+			t.Fatal(diff)
+		}
+
+		if diff := cmp.Diff([]*pilosa.StmtColumn{
+			{Name: "a", Type: "INT"},
+			{Name: "b", Type: "INT"},
+		}, columns); diff != "" {
+			t.Fatal(diff)
+		}
+	})
+
 	t.Run("ErrFieldNotFound", func(t *testing.T) {
 		_, err := c.GetNode(0).Server.PlanSQL(context.Background(), `SELECT xyz FROM i0`)
 		if err == nil || !strings.Contains(err.Error(), `xyz: field not found`) {
@@ -376,6 +393,39 @@ func TestPlanner_GroupBy(t *testing.T) {
 			t.Fatal(diff)
 		}
 	})
+
+	t.Run("ReorderColumns", func(t *testing.T) {
+		results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT x, COUNT(*) FROM i0 GROUP BY x`)
+		if diff := cmp.Diff([][]interface{}{
+			{int64(10), int64(2)},
+			{int64(20), int64(2)},
+		}, results); diff != "" {
+			t.Fatal(diff)
+		}
+
+		if diff := cmp.Diff([]*pilosa.StmtColumn{
+			{Name: "x", Type: "INT"},
+			{Name: "count", Type: "INT"},
+		}, columns); diff != "" {
+			t.Fatal(diff)
+		}
+	})
+
+	t.Run("NoResultColumn", func(t *testing.T) {
+		results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT COUNT(*) FROM i0 GROUP BY x`)
+		if diff := cmp.Diff([][]interface{}{
+			{int64(2)},
+			{int64(2)},
+		}, results); diff != "" {
+			t.Fatal(diff)
+		}
+
+		if diff := cmp.Diff([]*pilosa.StmtColumn{
+			{Name: "count", Type: "INT"},
+		}, columns); diff != "" {
+			t.Fatal(diff)
+		}
+	})
 }
 
 func mustQueryRows(tb testing.TB, svr *pilosa.Server, q string) (results [][]interface{}, columns []*pilosa.StmtColumn) {

From 95dc4a1a501ca72135c558e9530716ed11a3464e Mon Sep 17 00:00:00 2001
From: Todd Gruben <todd@pilosa.com>
Date: Mon, 13 Sep 2021 14:02:50 -0500
Subject: [PATCH 24/66] basic looker connection tests pass sql1 pass through
 works

---
 go.mod                    |   2 +-
 pg/lookerToFeaturebase.md | 652 ++++++++++++++++++++++++++++++++++++
 pg/lookerToPostgres.md    | 677 ++++++++++++++++++++++++++++++++++++++
 pg/message/io.go          |   2 +
 pg/message/message.go     | 181 +++++++++-
 pg/protocol.go            | 543 +++++++++++++++++++++++++++++-
 pg/query.go               |   1 +
 pg/server.go              |   5 +
 server/pg.go              |  49 ++-
 sql/mapper.go             |   6 +
 10 files changed, 2098 insertions(+), 20 deletions(-)
 create mode 100644 pg/lookerToFeaturebase.md
 create mode 100644 pg/lookerToPostgres.md

diff --git a/go.mod b/go.mod
index e590bdd1b..d64a32730 100644
--- a/go.mod
+++ b/go.mod
@@ -54,7 +54,7 @@ require (
 	golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect
 	golang.org/x/text v0.3.5 // indirect
 	google.golang.org/grpc v1.28.0
-	gopkg.in/yaml.v2 v2.3.0 // indirect
+	gopkg.in/yaml.v2 v2.3.0
 	modernc.org/mathutil v1.0.0
 	modernc.org/strutil v1.0.0
 	sigs.k8s.io/yaml v1.2.0 // indirect
diff --git a/pg/lookerToFeaturebase.md b/pg/lookerToFeaturebase.md
new file mode 100644
index 000000000..cd7c82b9b
--- /dev/null
+++ b/pg/lookerToFeaturebase.md
@@ -0,0 +1,652 @@
+```mermaid
+sequenceDiagram
+
+participant 213070643358480 as c0
+participant 213070643358482 as c1
+participant 213070643358484 as c2
+participant 213070643358486 as c3
+participant 213070643358488 as c4
+participant 213070643358490 as c5
+participant 213070643358492 as c6
+participant 213070643358494 as c7
+participant 213070643358496 as c8
+participant 213070643358498 as c9
+participant 213070643358500 as c10
+participant 213070643358502 as c11
+213070643358480->>server:+SSL REQUEST
+server-->>213070643358480:-SSL BACKEND ANSWER: N
+213070643358480->>server:+STARTUP MESSAGE version: 3  database=trait_store  extra_float_digits=2  TimeZone=GMT  client_encoding=UTF8  user=docker  DateStyle=ISO
+server-->>213070643358480:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
+server-->>213070643358480:-PARAMETER STATUS name='application_name', value=''
+server-->>213070643358480:-PARAMETER STATUS name='client_encoding', value='UTF8'
+server-->>213070643358480:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
+server-->>213070643358480:-PARAMETER STATUS name='integer_datetimes', value='on'
+server-->>213070643358480:-PARAMETER STATUS name='IntervalStyle', value='postgres'
+server-->>213070643358480:-PARAMETER STATUS name='is_superuser', value='on'
+server-->>213070643358480:-PARAMETER STATUS name='server_encoding', value='UTF8'
+server-->>213070643358480:-PARAMETER STATUS name='server_version', value='13.0.0'
+server-->>213070643358480:-PARAMETER STATUS name='session_authorization', value='docker'
+server-->>213070643358480:-PARAMETER STATUS name='standard_conforming_strings', value='on'
+server-->>213070643358480:-PARAMETER STATUS name='TimeZone', value='GMT'
+server-->>213070643358480:-BACKEND KEY DATA pid=1459324827, key=1506254533
+server-->>213070643358480:-READY FOR QUERY type=<IDLE>
+213070643358480->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
+213070643358480->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358480->>server:+EXECUTE name='', nb_rows=1
+213070643358480->>server:+SYNC
+server-->>213070643358480:-PARSE COMPLETE
+server-->>213070643358480:-BIND COMPLETE
+server-->>213070643358480:-COMMAND COMPLETE command='SET'
+server-->>213070643358480:-READY FOR QUERY type=<IDLE>
+213070643358480->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
+213070643358480->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358480->>server:+EXECUTE name='', nb_rows=1
+213070643358480->>server:+SYNC
+server-->>213070643358480:-PARSE COMPLETE
+server-->>213070643358480:-BIND COMPLETE
+server-->>213070643358480:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
+server-->>213070643358480:-COMMAND COMPLETE command='SET'
+server-->>213070643358480:-READY FOR QUERY type=<IDLE>
+213070643358480->>server:+PARSE name='', num_params=0, params_type=, query=
+213070643358480->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358480->>server:+DESCRIBE kind='P', name=''
+213070643358480->>server:+EXECUTE name='', nb_rows=1
+213070643358480->>server:+SYNC
+server-->>213070643358480:-PARSE COMPLETE
+server-->>213070643358480:-BIND COMPLETE
+server-->>213070643358480:-NO DATA
+server-->>213070643358480:-EMPTY QUERY RESPONSE
+server-->>213070643358480:-READY FOR QUERY type=<IDLE>
+213070643358480->>server:+DISCONNECT
+213070643358482->>server:+SSL REQUEST
+server-->>213070643358482:-SSL BACKEND ANSWER: N
+213070643358482->>server:+STARTUP MESSAGE version: 3  database=trait_store  extra_float_digits=2  TimeZone=GMT  client_encoding=UTF8  user=docker  DateStyle=ISO
+server-->>213070643358482:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
+server-->>213070643358482:-PARAMETER STATUS name='application_name', value=''
+server-->>213070643358482:-PARAMETER STATUS name='client_encoding', value='UTF8'
+server-->>213070643358482:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
+server-->>213070643358482:-PARAMETER STATUS name='integer_datetimes', value='on'
+server-->>213070643358482:-PARAMETER STATUS name='IntervalStyle', value='postgres'
+server-->>213070643358482:-PARAMETER STATUS name='is_superuser', value='on'
+server-->>213070643358482:-PARAMETER STATUS name='server_encoding', value='UTF8'
+server-->>213070643358482:-PARAMETER STATUS name='server_version', value='13.0.0'
+server-->>213070643358482:-PARAMETER STATUS name='session_authorization', value='docker'
+server-->>213070643358482:-PARAMETER STATUS name='standard_conforming_strings', value='on'
+server-->>213070643358482:-PARAMETER STATUS name='TimeZone', value='GMT'
+server-->>213070643358482:-BACKEND KEY DATA pid=1742342691, key=1299317425
+server-->>213070643358482:-READY FOR QUERY type=<IDLE>
+213070643358482->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
+213070643358482->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358482->>server:+EXECUTE name='', nb_rows=1
+213070643358482->>server:+SYNC
+server-->>213070643358482:-PARSE COMPLETE
+server-->>213070643358482:-BIND COMPLETE
+server-->>213070643358482:-COMMAND COMPLETE command='SET'
+server-->>213070643358482:-READY FOR QUERY type=<IDLE>
+213070643358482->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
+213070643358482->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358482->>server:+EXECUTE name='', nb_rows=1
+213070643358482->>server:+SYNC
+server-->>213070643358482:-PARSE COMPLETE
+server-->>213070643358482:-BIND COMPLETE
+server-->>213070643358482:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
+server-->>213070643358482:-COMMAND COMPLETE command='SET'
+server-->>213070643358482:-READY FOR QUERY type=<IDLE>
+213070643358482->>server:+PARSE name='', num_params=0, params_type=, query=
+213070643358482->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358482->>server:+DESCRIBE kind='P', name=''
+213070643358482->>server:+EXECUTE name='', nb_rows=1
+213070643358482->>server:+SYNC
+server-->>213070643358482:-PARSE COMPLETE
+server-->>213070643358482:-BIND COMPLETE
+server-->>213070643358482:-NO DATA
+server-->>213070643358482:-EMPTY QUERY RESPONSE
+server-->>213070643358482:-READY FOR QUERY type=<IDLE>
+213070643358482->>server:+DISCONNECT
+213070643358484->>server:+SSL REQUEST
+server-->>213070643358484:-SSL BACKEND ANSWER: N
+213070643358484->>server:+STARTUP MESSAGE version: 3  database=trait_store  extra_float_digits=2  TimeZone=GMT  client_encoding=UTF8  user=docker  DateStyle=ISO
+server-->>213070643358484:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
+server-->>213070643358484:-PARAMETER STATUS name='application_name', value=''
+server-->>213070643358484:-PARAMETER STATUS name='client_encoding', value='UTF8'
+server-->>213070643358484:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
+server-->>213070643358484:-PARAMETER STATUS name='integer_datetimes', value='on'
+server-->>213070643358484:-PARAMETER STATUS name='IntervalStyle', value='postgres'
+server-->>213070643358484:-PARAMETER STATUS name='is_superuser', value='on'
+server-->>213070643358484:-PARAMETER STATUS name='server_encoding', value='UTF8'
+server-->>213070643358484:-PARAMETER STATUS name='server_version', value='13.0.0'
+server-->>213070643358484:-PARAMETER STATUS name='session_authorization', value='docker'
+server-->>213070643358484:-PARAMETER STATUS name='standard_conforming_strings', value='on'
+server-->>213070643358484:-PARAMETER STATUS name='TimeZone', value='GMT'
+server-->>213070643358484:-BACKEND KEY DATA pid=700852804, key=1377869267
+server-->>213070643358484:-READY FOR QUERY type=<IDLE>
+213070643358484->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
+213070643358484->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358484->>server:+EXECUTE name='', nb_rows=1
+213070643358484->>server:+SYNC
+server-->>213070643358484:-PARSE COMPLETE
+server-->>213070643358484:-BIND COMPLETE
+server-->>213070643358484:-COMMAND COMPLETE command='SET'
+server-->>213070643358484:-READY FOR QUERY type=<IDLE>
+213070643358484->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
+213070643358484->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358484->>server:+EXECUTE name='', nb_rows=1
+213070643358484->>server:+SYNC
+server-->>213070643358484:-PARSE COMPLETE
+server-->>213070643358484:-BIND COMPLETE
+server-->>213070643358484:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
+server-->>213070643358484:-COMMAND COMPLETE command='SET'
+server-->>213070643358484:-READY FOR QUERY type=<IDLE>
+213070643358484->>server:+PARSE name='', num_params=0, params_type=, query=
+213070643358484->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358484->>server:+DESCRIBE kind='P', name=''
+213070643358484->>server:+EXECUTE name='', nb_rows=1
+213070643358484->>server:+SYNC
+server-->>213070643358484:-PARSE COMPLETE
+server-->>213070643358484:-BIND COMPLETE
+server-->>213070643358484:-NO DATA
+server-->>213070643358484:-EMPTY QUERY RESPONSE
+server-->>213070643358484:-READY FOR QUERY type=<IDLE>
+213070643358484->>server:+DISCONNECT
+213070643358486->>server:+SSL REQUEST
+server-->>213070643358486:-SSL BACKEND ANSWER: N
+213070643358486->>server:+STARTUP MESSAGE version: 3  database=trait_store  extra_float_digits=2  TimeZone=GMT  client_encoding=UTF8  user=docker  DateStyle=ISO
+server-->>213070643358486:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
+server-->>213070643358486:-PARAMETER STATUS name='application_name', value=''
+server-->>213070643358486:-PARAMETER STATUS name='client_encoding', value='UTF8'
+server-->>213070643358486:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
+server-->>213070643358486:-PARAMETER STATUS name='integer_datetimes', value='on'
+server-->>213070643358486:-PARAMETER STATUS name='IntervalStyle', value='postgres'
+server-->>213070643358486:-PARAMETER STATUS name='is_superuser', value='on'
+server-->>213070643358486:-PARAMETER STATUS name='server_encoding', value='UTF8'
+server-->>213070643358486:-PARAMETER STATUS name='server_version', value='13.0.0'
+server-->>213070643358486:-PARAMETER STATUS name='session_authorization', value='docker'
+server-->>213070643358486:-PARAMETER STATUS name='standard_conforming_strings', value='on'
+server-->>213070643358486:-PARAMETER STATUS name='TimeZone', value='GMT'
+server-->>213070643358486:-BACKEND KEY DATA pid=413235241, key=1302652759
+server-->>213070643358486:-READY FOR QUERY type=<IDLE>
+213070643358486->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
+213070643358486->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358486->>server:+EXECUTE name='', nb_rows=1
+213070643358486->>server:+SYNC
+server-->>213070643358486:-PARSE COMPLETE
+server-->>213070643358486:-BIND COMPLETE
+server-->>213070643358486:-COMMAND COMPLETE command='SET'
+server-->>213070643358486:-READY FOR QUERY type=<IDLE>
+213070643358486->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
+213070643358486->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358486->>server:+EXECUTE name='', nb_rows=1
+213070643358486->>server:+SYNC
+server-->>213070643358486:-PARSE COMPLETE
+server-->>213070643358486:-BIND COMPLETE
+server-->>213070643358486:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
+server-->>213070643358486:-COMMAND COMPLETE command='SET'
+server-->>213070643358486:-READY FOR QUERY type=<IDLE>
+213070643358486->>server:+PARSE name='', num_params=0, params_type=, query=
+213070643358486->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358486->>server:+DESCRIBE kind='P', name=''
+213070643358486->>server:+EXECUTE name='', nb_rows=1
+213070643358486->>server:+SYNC
+server-->>213070643358486:-PARSE COMPLETE
+server-->>213070643358486:-BIND COMPLETE
+server-->>213070643358486:-NO DATA
+server-->>213070643358486:-EMPTY QUERY RESPONSE
+server-->>213070643358486:-READY FOR QUERY type=<IDLE>
+213070643358486->>server:+PARSE name='', num_params=0, params_type=, query=SELECT pg_backend_pid()
+213070643358486->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358486->>server:+DESCRIBE kind='P', name=''
+213070643358486->>server:+EXECUTE name='', nb_rows=0
+213070643358486->>server:+SYNC
+server-->>213070643358486:-PARSE COMPLETE
+server-->>213070643358486:-BIND COMPLETE
+server-->>213070643358486:-ROW DESCRIPTION: num_fields=1  ---[Field 01]---  name='pg_backend_pid'  type=23  type_len=4  type_mod=4294967295  relid=0  attnum=0  format=0
+server-->>213070643358486:-DATA ROW num_values=1  ---[Value 0001]---  length=9  value='413235241'
+server-->>213070643358486:-COMMAND COMPLETE command='SELECT'
+server-->>213070643358486:-READY FOR QUERY type=<IDLE>
+213070643358486->>server:+PARSE name='', num_params=0, params_type=, query=SELECT VERSION() AS version
+213070643358486->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358486->>server:+DESCRIBE kind='P', name=''
+213070643358486->>server:+EXECUTE name='', nb_rows=0
+213070643358486->>server:+SYNC
+server-->>213070643358486:-PARSE COMPLETE
+server-->>213070643358486:-BIND COMPLETE
+server-->>213070643358486:-ROW DESCRIPTION: num_fields=1  ---[Field 01]---  name='version'  type=25  type_len=65535  type_mod=4294967295  relid=0  attnum=0  format=0
+server-->>213070643358486:-DATA ROW num_values=1  ---[Value 0001]---  length=27  value='PostgresSQL 13.0 (molecula)'
+server-->>213070643358486:-COMMAND COMPLETE command='SELECT'
+server-->>213070643358486:-READY FOR QUERY type=<IDLE>
+213070643358486->>server:+PARSE name='', num_params=0, params_type=, query=        SELECT COUNT(*)        FROM pg_type AS t0,             pg_aggregate AS t1,             pg_settings AS t2,             pg_settings AS t3
+213070643358486->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358486->>server:+DESCRIBE kind='P', name=''
+213070643358486->>server:+EXECUTE name='', nb_rows=0
+213070643358486->>server:+SYNC
+213070643358488->>server:+SSL REQUEST
+server-->>213070643358488:-SSL BACKEND ANSWER: N
+213070643358488->>server:+STARTUP MESSAGE version: 3  database=trait_store  extra_float_digits=2  TimeZone=GMT  client_encoding=UTF8  user=docker  DateStyle=ISO
+server-->>213070643358488:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
+server-->>213070643358488:-PARAMETER STATUS name='application_name', value=''
+server-->>213070643358488:-PARAMETER STATUS name='client_encoding', value='UTF8'
+server-->>213070643358488:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
+server-->>213070643358488:-PARAMETER STATUS name='integer_datetimes', value='on'
+server-->>213070643358488:-PARAMETER STATUS name='IntervalStyle', value='postgres'
+server-->>213070643358488:-PARAMETER STATUS name='is_superuser', value='on'
+server-->>213070643358488:-PARAMETER STATUS name='server_encoding', value='UTF8'
+server-->>213070643358488:-PARAMETER STATUS name='server_version', value='13.0.0'
+server-->>213070643358488:-PARAMETER STATUS name='session_authorization', value='docker'
+server-->>213070643358488:-PARAMETER STATUS name='standard_conforming_strings', value='on'
+server-->>213070643358488:-PARAMETER STATUS name='TimeZone', value='GMT'
+server-->>213070643358488:-BACKEND KEY DATA pid=180554708, key=1504602717
+server-->>213070643358488:-READY FOR QUERY type=<IDLE>
+213070643358488->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
+213070643358488->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358488->>server:+EXECUTE name='', nb_rows=1
+213070643358488->>server:+SYNC
+server-->>213070643358488:-PARSE COMPLETE
+server-->>213070643358488:-BIND COMPLETE
+server-->>213070643358488:-COMMAND COMPLETE command='SET'
+server-->>213070643358488:-READY FOR QUERY type=<IDLE>
+213070643358488->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
+213070643358488->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358488->>server:+EXECUTE name='', nb_rows=1
+213070643358488->>server:+SYNC
+server-->>213070643358488:-PARSE COMPLETE
+server-->>213070643358488:-BIND COMPLETE
+server-->>213070643358488:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
+server-->>213070643358488:-COMMAND COMPLETE command='SET'
+server-->>213070643358488:-READY FOR QUERY type=<IDLE>
+213070643358488->>server:+PARSE name='', num_params=0, params_type=, query=
+213070643358488->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358488->>server:+DESCRIBE kind='P', name=''
+213070643358488->>server:+EXECUTE name='', nb_rows=1
+213070643358488->>server:+SYNC
+server-->>213070643358488:-PARSE COMPLETE
+server-->>213070643358488:-BIND COMPLETE
+server-->>213070643358488:-NO DATA
+server-->>213070643358488:-EMPTY QUERY RESPONSE
+server-->>213070643358488:-READY FOR QUERY type=<IDLE>
+213070643358488->>server:+DISCONNECT
+213070643358490->>server:+SSL REQUEST
+server-->>213070643358490:-SSL BACKEND ANSWER: N
+213070643358490->>server:+STARTUP MESSAGE version: 3  database=trait_store  extra_float_digits=2  TimeZone=GMT  client_encoding=UTF8  user=docker  DateStyle=ISO
+server-->>213070643358490:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
+server-->>213070643358490:-PARAMETER STATUS name='application_name', value=''
+server-->>213070643358490:-PARAMETER STATUS name='client_encoding', value='UTF8'
+server-->>213070643358490:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
+server-->>213070643358490:-PARAMETER STATUS name='integer_datetimes', value='on'
+server-->>213070643358490:-PARAMETER STATUS name='IntervalStyle', value='postgres'
+server-->>213070643358490:-PARAMETER STATUS name='is_superuser', value='on'
+server-->>213070643358490:-PARAMETER STATUS name='server_encoding', value='UTF8'
+server-->>213070643358490:-PARAMETER STATUS name='server_version', value='13.0.0'
+server-->>213070643358490:-PARAMETER STATUS name='session_authorization', value='docker'
+server-->>213070643358490:-PARAMETER STATUS name='standard_conforming_strings', value='on'
+server-->>213070643358490:-PARAMETER STATUS name='TimeZone', value='GMT'
+server-->>213070643358490:-BACKEND KEY DATA pid=1713101217, key=29850369
+server-->>213070643358490:-READY FOR QUERY type=<IDLE>
+213070643358490->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
+213070643358490->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358490->>server:+EXECUTE name='', nb_rows=1
+213070643358490->>server:+SYNC
+server-->>213070643358490:-PARSE COMPLETE
+server-->>213070643358490:-BIND COMPLETE
+server-->>213070643358490:-COMMAND COMPLETE command='SET'
+server-->>213070643358490:-READY FOR QUERY type=<IDLE>
+213070643358490->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
+213070643358490->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358490->>server:+EXECUTE name='', nb_rows=1
+213070643358490->>server:+SYNC
+server-->>213070643358490:-PARSE COMPLETE
+server-->>213070643358490:-BIND COMPLETE
+server-->>213070643358490:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
+server-->>213070643358490:-COMMAND COMPLETE command='SET'
+server-->>213070643358490:-READY FOR QUERY type=<IDLE>
+213070643358490->>server:+PARSE name='', num_params=0, params_type=, query=
+213070643358490->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358490->>server:+DESCRIBE kind='P', name=''
+213070643358490->>server:+EXECUTE name='', nb_rows=1
+213070643358490->>server:+SYNC
+server-->>213070643358490:-PARSE COMPLETE
+server-->>213070643358490:-BIND COMPLETE
+server-->>213070643358490:-NO DATA
+server-->>213070643358490:-EMPTY QUERY RESPONSE
+server-->>213070643358490:-READY FOR QUERY type=<IDLE>
+213070643358490->>server:+PARSE name='', num_params=0, params_type=, query=SELECT VERSION() AS version
+213070643358490->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358490->>server:+DESCRIBE kind='P', name=''
+213070643358490->>server:+EXECUTE name='', nb_rows=0
+213070643358490->>server:+SYNC
+server-->>213070643358490:-PARSE COMPLETE
+server-->>213070643358490:-BIND COMPLETE
+server-->>213070643358490:-ROW DESCRIPTION: num_fields=1  ---[Field 01]---  name='version'  type=25  type_len=65535  type_mod=4294967295  relid=0  attnum=0  format=0
+server-->>213070643358490:-DATA ROW num_values=1  ---[Value 0001]---  length=27  value='PostgresSQL 13.0 (molecula)'
+server-->>213070643358490:-COMMAND COMPLETE command='SELECT'
+server-->>213070643358490:-READY FOR QUERY type=<IDLE>
+213070643358490->>server:+PARSE name='', num_params=0, params_type=, query=
+213070643358490->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358490->>server:+DESCRIBE kind='P', name=''
+213070643358490->>server:+EXECUTE name='', nb_rows=1
+213070643358490->>server:+SYNC
+server-->>213070643358490:-PARSE COMPLETE
+server-->>213070643358490:-BIND COMPLETE
+server-->>213070643358490:-NO DATA
+server-->>213070643358490:-EMPTY QUERY RESPONSE
+server-->>213070643358490:-READY FOR QUERY type=<IDLE>
+213070643358490->>server:+PARSE name='', num_params=0, params_type=, query=          SELECT pid as id,            query as stmt,            EXTRACT(seconds from query_start - NOW()) as elapsed_time          FROM pg_stat_activity          WHERE usename='docker'
+213070643358490->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358490->>server:+DESCRIBE kind='P', name=''
+213070643358490->>server:+EXECUTE name='', nb_rows=0
+213070643358490->>server:+SYNC
+server-->>213070643358490:-PARSE COMPLETE
+server-->>213070643358490:-BIND COMPLETE
+server-->>213070643358490:-ROW DESCRIPTION: num_fields=3  ---[Field 01]---  name='id'  type=23  type_len=4  type_mod=4294967295  relid=0  attnum=0  format=0  ---[Field 02]---  name='stmt'  type=25  type_len=65535  type_mod=4294967295  relid=0  attnum=0  format=0  ---[Field 03]---  name='elapsed_time'  type=701  type_len=8  type_mod=4294967295  relid=0  attnum=0  format=0
+server-->>213070643358490:-DATA ROW num_values=3  ---[Value 0001]---  length=9  value='413235241'  ---[Value 0002]---  length=148  value='        SELECT COUNT(*).        FROM pg_type AS t0,.             pg_aggregate AS t1,.             pg_settings AS t2,.             pg_settings AS t3.'  ---[Value 0003]---  length=11  value='1.311371547'
+server-->>213070643358490:-DATA ROW num_values=3  ---[Value 0001]---  length=10  value='1713101217'  ---[Value 0002]---  length=190  value='          SELECT pid as id,.            query as stmt,.            EXTRACT(seconds from query_start - NOW()) as elapsed_time.          FROM pg_stat_activity.          WHERE usename='docker'.'  ---[Value 0003]---  length=11  value='0.000169969'
+server-->>213070643358490:-COMMAND COMPLETE command='SELECT'
+server-->>213070643358490:-READY FOR QUERY type=<IDLE>
+213070643358492->>server:+SSL REQUEST
+server-->>213070643358492:-SSL BACKEND ANSWER: N
+213070643358492->>server:+STARTUP MESSAGE version: 3  database=trait_store  extra_float_digits=2  TimeZone=GMT  client_encoding=UTF8  user=docker  DateStyle=ISO
+server-->>213070643358492:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
+server-->>213070643358492:-PARAMETER STATUS name='application_name', value=''
+server-->>213070643358492:-PARAMETER STATUS name='client_encoding', value='UTF8'
+server-->>213070643358492:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
+server-->>213070643358492:-PARAMETER STATUS name='integer_datetimes', value='on'
+server-->>213070643358492:-PARAMETER STATUS name='IntervalStyle', value='postgres'
+server-->>213070643358492:-PARAMETER STATUS name='is_superuser', value='on'
+server-->>213070643358492:-PARAMETER STATUS name='server_encoding', value='UTF8'
+server-->>213070643358492:-PARAMETER STATUS name='server_version', value='13.0.0'
+server-->>213070643358492:-PARAMETER STATUS name='session_authorization', value='docker'
+server-->>213070643358492:-PARAMETER STATUS name='standard_conforming_strings', value='on'
+server-->>213070643358492:-PARAMETER STATUS name='TimeZone', value='GMT'
+server-->>213070643358492:-BACKEND KEY DATA pid=2034361563, key=1419995666
+server-->>213070643358492:-READY FOR QUERY type=<IDLE>
+213070643358492->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
+213070643358492->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358492->>server:+EXECUTE name='', nb_rows=1
+213070643358492->>server:+SYNC
+server-->>213070643358492:-PARSE COMPLETE
+server-->>213070643358492:-BIND COMPLETE
+server-->>213070643358492:-COMMAND COMPLETE command='SET'
+server-->>213070643358492:-READY FOR QUERY type=<IDLE>
+213070643358492->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
+213070643358492->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358492->>server:+EXECUTE name='', nb_rows=1
+213070643358492->>server:+SYNC
+server-->>213070643358492:-PARSE COMPLETE
+server-->>213070643358492:-BIND COMPLETE
+server-->>213070643358492:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
+server-->>213070643358492:-COMMAND COMPLETE command='SET'
+server-->>213070643358492:-READY FOR QUERY type=<IDLE>
+213070643358492->>server:+PARSE name='', num_params=0, params_type=, query=
+213070643358492->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358492->>server:+DESCRIBE kind='P', name=''
+213070643358492->>server:+EXECUTE name='', nb_rows=1
+213070643358492->>server:+SYNC
+server-->>213070643358492:-PARSE COMPLETE
+server-->>213070643358492:-BIND COMPLETE
+server-->>213070643358492:-NO DATA
+server-->>213070643358492:-EMPTY QUERY RESPONSE
+server-->>213070643358492:-READY FOR QUERY type=<IDLE>
+213070643358492->>server:+DISCONNECT
+213070643358494->>server:+SSL REQUEST
+server-->>213070643358494:-SSL BACKEND ANSWER: N
+213070643358494->>server:+STARTUP MESSAGE version: 3  database=trait_store  extra_float_digits=2  TimeZone=GMT  client_encoding=UTF8  user=docker  DateStyle=ISO
+server-->>213070643358494:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
+server-->>213070643358494:-PARAMETER STATUS name='application_name', value=''
+server-->>213070643358494:-PARAMETER STATUS name='client_encoding', value='UTF8'
+server-->>213070643358494:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
+server-->>213070643358494:-PARAMETER STATUS name='integer_datetimes', value='on'
+server-->>213070643358494:-PARAMETER STATUS name='IntervalStyle', value='postgres'
+server-->>213070643358494:-PARAMETER STATUS name='is_superuser', value='on'
+server-->>213070643358494:-PARAMETER STATUS name='server_encoding', value='UTF8'
+server-->>213070643358494:-PARAMETER STATUS name='server_version', value='13.0.0'
+server-->>213070643358494:-PARAMETER STATUS name='session_authorization', value='docker'
+server-->>213070643358494:-PARAMETER STATUS name='standard_conforming_strings', value='on'
+server-->>213070643358494:-PARAMETER STATUS name='TimeZone', value='GMT'
+server-->>213070643358494:-BACKEND KEY DATA pid=964429462, key=1673405115
+server-->>213070643358494:-READY FOR QUERY type=<IDLE>
+213070643358494->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
+213070643358494->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358494->>server:+EXECUTE name='', nb_rows=1
+213070643358494->>server:+SYNC
+server-->>213070643358494:-PARSE COMPLETE
+server-->>213070643358494:-BIND COMPLETE
+server-->>213070643358494:-COMMAND COMPLETE command='SET'
+server-->>213070643358494:-READY FOR QUERY type=<IDLE>
+213070643358494->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
+213070643358494->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358494->>server:+EXECUTE name='', nb_rows=1
+213070643358494->>server:+SYNC
+server-->>213070643358494:-PARSE COMPLETE
+server-->>213070643358494:-BIND COMPLETE
+server-->>213070643358494:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
+server-->>213070643358494:-COMMAND COMPLETE command='SET'
+server-->>213070643358494:-READY FOR QUERY type=<IDLE>
+213070643358494->>server:+PARSE name='', num_params=0, params_type=, query=
+213070643358494->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358494->>server:+DESCRIBE kind='P', name=''
+213070643358494->>server:+EXECUTE name='', nb_rows=1
+213070643358494->>server:+SYNC
+server-->>213070643358494:-PARSE COMPLETE
+server-->>213070643358494:-BIND COMPLETE
+server-->>213070643358494:-NO DATA
+server-->>213070643358494:-EMPTY QUERY RESPONSE
+server-->>213070643358494:-READY FOR QUERY type=<IDLE>
+213070643358494->>server:+PARSE name='', num_params=0, params_type=, query=select pg_terminate_backend(413235241)
+213070643358494->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358494->>server:+DESCRIBE kind='P', name=''
+213070643358494->>server:+EXECUTE name='', nb_rows=0
+213070643358494->>server:+SYNC
+server-->>213070643358494:-PARSE COMPLETE
+server-->>213070643358494:-BIND COMPLETE
+server-->>213070643358494:-ROW DESCRIPTION: num_fields=1  ---[Field 01]---  name='pg_terminate_backend'  type=16  type_len=1  type_mod=4294967295  relid=0  attnum=0  format=0
+server-->>213070643358494:-DATA ROW num_values=1  ---[Value 0001]---  length=1  value='t'
+server-->>213070643358494:-COMMAND COMPLETE command='SELECT 1'
+server-->>213070643358486:-PARSE COMPLETE
+server-->>213070643358486:-BIND COMPLETE
+server-->>213070643358486:-ROW DESCRIPTION: num_fields=1  ---[Field 01]---  name='count'  type=20  type_len=8  type_mod=4294967295  relid=0  attnum=0  format=0
+server-->>213070643358486:-ERROR RESPONSE  Severity: 'FATAL'  Message: 'terminating connection due to administrator command'  Code: '57P01'
+server-->>213070643358494:-READY FOR QUERY type=<IDLE>
+213070643358490->>server:+DISCONNECT
+213070643358494->>server:+DISCONNECT
+213070643358496->>server:+SSL REQUEST
+server-->>213070643358496:-SSL BACKEND ANSWER: N
+213070643358496->>server:+STARTUP MESSAGE version: 3  database=trait_store  extra_float_digits=2  TimeZone=GMT  client_encoding=UTF8  user=docker  DateStyle=ISO
+server-->>213070643358496:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
+server-->>213070643358496:-PARAMETER STATUS name='application_name', value=''
+server-->>213070643358496:-PARAMETER STATUS name='client_encoding', value='UTF8'
+server-->>213070643358496:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
+server-->>213070643358496:-PARAMETER STATUS name='integer_datetimes', value='on'
+server-->>213070643358496:-PARAMETER STATUS name='IntervalStyle', value='postgres'
+server-->>213070643358496:-PARAMETER STATUS name='is_superuser', value='on'
+server-->>213070643358496:-PARAMETER STATUS name='server_encoding', value='UTF8'
+server-->>213070643358496:-PARAMETER STATUS name='server_version', value='13.0.0'
+server-->>213070643358496:-PARAMETER STATUS name='session_authorization', value='docker'
+server-->>213070643358496:-PARAMETER STATUS name='standard_conforming_strings', value='on'
+server-->>213070643358496:-PARAMETER STATUS name='TimeZone', value='GMT'
+server-->>213070643358496:-BACKEND KEY DATA pid=841257432, key=992978867
+server-->>213070643358496:-READY FOR QUERY type=<IDLE>
+213070643358496->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
+213070643358496->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358496->>server:+EXECUTE name='', nb_rows=1
+213070643358496->>server:+SYNC
+server-->>213070643358496:-PARSE COMPLETE
+server-->>213070643358496:-BIND COMPLETE
+server-->>213070643358496:-COMMAND COMPLETE command='SET'
+server-->>213070643358496:-READY FOR QUERY type=<IDLE>
+213070643358496->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
+213070643358496->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358496->>server:+EXECUTE name='', nb_rows=1
+213070643358496->>server:+SYNC
+server-->>213070643358496:-PARSE COMPLETE
+server-->>213070643358496:-BIND COMPLETE
+server-->>213070643358496:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
+server-->>213070643358496:-COMMAND COMPLETE command='SET'
+server-->>213070643358496:-READY FOR QUERY type=<IDLE>
+213070643358496->>server:+PARSE name='', num_params=0, params_type=, query=
+213070643358496->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358496->>server:+DESCRIBE kind='P', name=''
+213070643358496->>server:+EXECUTE name='', nb_rows=1
+213070643358496->>server:+SYNC
+server-->>213070643358496:-PARSE COMPLETE
+server-->>213070643358496:-BIND COMPLETE
+server-->>213070643358496:-NO DATA
+server-->>213070643358496:-EMPTY QUERY RESPONSE
+server-->>213070643358496:-READY FOR QUERY type=<IDLE>
+213070643358496->>server:+DISCONNECT
+213070643358498->>server:+SSL REQUEST
+server-->>213070643358498:-SSL BACKEND ANSWER: N
+213070643358498->>server:+STARTUP MESSAGE version: 3  database=trait_store  extra_float_digits=2  TimeZone=GMT  client_encoding=UTF8  user=docker  DateStyle=ISO
+server-->>213070643358498:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
+server-->>213070643358498:-PARAMETER STATUS name='application_name', value=''
+server-->>213070643358498:-PARAMETER STATUS name='client_encoding', value='UTF8'
+server-->>213070643358498:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
+server-->>213070643358498:-PARAMETER STATUS name='integer_datetimes', value='on'
+server-->>213070643358498:-PARAMETER STATUS name='IntervalStyle', value='postgres'
+server-->>213070643358498:-PARAMETER STATUS name='is_superuser', value='on'
+server-->>213070643358498:-PARAMETER STATUS name='server_encoding', value='UTF8'
+server-->>213070643358498:-PARAMETER STATUS name='server_version', value='13.0.0'
+server-->>213070643358498:-PARAMETER STATUS name='session_authorization', value='docker'
+server-->>213070643358498:-PARAMETER STATUS name='standard_conforming_strings', value='on'
+server-->>213070643358498:-PARAMETER STATUS name='TimeZone', value='GMT'
+server-->>213070643358498:-BACKEND KEY DATA pid=645931978, key=579078180
+server-->>213070643358498:-READY FOR QUERY type=<IDLE>
+213070643358498->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
+213070643358498->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358498->>server:+EXECUTE name='', nb_rows=1
+213070643358498->>server:+SYNC
+server-->>213070643358498:-PARSE COMPLETE
+server-->>213070643358498:-BIND COMPLETE
+server-->>213070643358498:-COMMAND COMPLETE command='SET'
+server-->>213070643358498:-READY FOR QUERY type=<IDLE>
+213070643358498->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
+213070643358498->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358498->>server:+EXECUTE name='', nb_rows=1
+213070643358498->>server:+SYNC
+server-->>213070643358498:-PARSE COMPLETE
+server-->>213070643358498:-BIND COMPLETE
+server-->>213070643358498:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
+server-->>213070643358498:-COMMAND COMPLETE command='SET'
+server-->>213070643358498:-READY FOR QUERY type=<IDLE>
+213070643358498->>server:+PARSE name='', num_params=0, params_type=, query=
+213070643358498->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358498->>server:+DESCRIBE kind='P', name=''
+213070643358498->>server:+EXECUTE name='', nb_rows=1
+213070643358498->>server:+SYNC
+server-->>213070643358498:-PARSE COMPLETE
+server-->>213070643358498:-BIND COMPLETE
+server-->>213070643358498:-NO DATA
+server-->>213070643358498:-EMPTY QUERY RESPONSE
+server-->>213070643358498:-READY FOR QUERY type=<IDLE>
+213070643358498->>server:+PARSE name='', num_params=0, params_type=, query=SELECT 1 
+213070643358498->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358498->>server:+DESCRIBE kind='P', name=''
+213070643358498->>server:+EXECUTE name='', nb_rows=0
+213070643358498->>server:+SYNC
+server-->>213070643358498:-PARSE COMPLETE
+server-->>213070643358498:-BIND COMPLETE
+server-->>213070643358498:-ROW DESCRIPTION: num_fields=1  ---[Field 01]---  name='?column?'  type=23  type_len=4  type_mod=4294967295  relid=0  attnum=0  format=0
+server-->>213070643358498:-DATA ROW num_values=1  ---[Value 0001]---  length=1  value='1'
+server-->>213070643358498:-COMMAND COMPLETE command='SELECT'
+server-->>213070643358498:-READY FOR QUERY type=<IDLE>
+213070643358498->>server:+DISCONNECT
+213070643358500->>server:+SSL REQUEST
+server-->>213070643358500:-SSL BACKEND ANSWER: N
+213070643358500->>server:+STARTUP MESSAGE version: 3  database=trait_store  extra_float_digits=2  TimeZone=GMT  client_encoding=UTF8  user=docker  DateStyle=ISO
+server-->>213070643358500:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
+server-->>213070643358500:-PARAMETER STATUS name='application_name', value=''
+server-->>213070643358500:-PARAMETER STATUS name='client_encoding', value='UTF8'
+server-->>213070643358500:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
+server-->>213070643358500:-PARAMETER STATUS name='integer_datetimes', value='on'
+server-->>213070643358500:-PARAMETER STATUS name='IntervalStyle', value='postgres'
+server-->>213070643358500:-PARAMETER STATUS name='is_superuser', value='on'
+server-->>213070643358500:-PARAMETER STATUS name='server_encoding', value='UTF8'
+server-->>213070643358500:-PARAMETER STATUS name='server_version', value='13.0.0'
+server-->>213070643358500:-PARAMETER STATUS name='session_authorization', value='docker'
+server-->>213070643358500:-PARAMETER STATUS name='standard_conforming_strings', value='on'
+server-->>213070643358500:-PARAMETER STATUS name='TimeZone', value='GMT'
+server-->>213070643358500:-BACKEND KEY DATA pid=1632401438, key=1341645778
+server-->>213070643358500:-READY FOR QUERY type=<IDLE>
+213070643358500->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
+213070643358500->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358500->>server:+EXECUTE name='', nb_rows=1
+213070643358500->>server:+SYNC
+server-->>213070643358500:-PARSE COMPLETE
+server-->>213070643358500:-BIND COMPLETE
+server-->>213070643358500:-COMMAND COMPLETE command='SET'
+server-->>213070643358500:-READY FOR QUERY type=<IDLE>
+213070643358500->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
+213070643358500->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358500->>server:+EXECUTE name='', nb_rows=1
+213070643358500->>server:+SYNC
+server-->>213070643358500:-PARSE COMPLETE
+server-->>213070643358500:-BIND COMPLETE
+server-->>213070643358500:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
+server-->>213070643358500:-COMMAND COMPLETE command='SET'
+server-->>213070643358500:-READY FOR QUERY type=<IDLE>
+213070643358500->>server:+PARSE name='', num_params=0, params_type=, query=
+213070643358500->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358500->>server:+DESCRIBE kind='P', name=''
+213070643358500->>server:+EXECUTE name='', nb_rows=1
+213070643358500->>server:+SYNC
+server-->>213070643358500:-PARSE COMPLETE
+server-->>213070643358500:-BIND COMPLETE
+server-->>213070643358500:-NO DATA
+server-->>213070643358500:-EMPTY QUERY RESPONSE
+server-->>213070643358500:-READY FOR QUERY type=<IDLE>
+213070643358500->>server:+DISCONNECT
+213070643358502->>server:+SSL REQUEST
+server-->>213070643358502:-SSL BACKEND ANSWER: N
+213070643358502->>server:+STARTUP MESSAGE version: 3  database=trait_store  extra_float_digits=2  TimeZone=GMT  client_encoding=UTF8  user=docker  DateStyle=ISO
+server-->>213070643358502:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
+server-->>213070643358502:-PARAMETER STATUS name='application_name', value=''
+server-->>213070643358502:-PARAMETER STATUS name='client_encoding', value='UTF8'
+server-->>213070643358502:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
+server-->>213070643358502:-PARAMETER STATUS name='integer_datetimes', value='on'
+server-->>213070643358502:-PARAMETER STATUS name='IntervalStyle', value='postgres'
+server-->>213070643358502:-PARAMETER STATUS name='is_superuser', value='on'
+server-->>213070643358502:-PARAMETER STATUS name='server_encoding', value='UTF8'
+server-->>213070643358502:-PARAMETER STATUS name='server_version', value='13.0.0'
+server-->>213070643358502:-PARAMETER STATUS name='session_authorization', value='docker'
+server-->>213070643358502:-PARAMETER STATUS name='standard_conforming_strings', value='on'
+server-->>213070643358502:-PARAMETER STATUS name='TimeZone', value='GMT'
+server-->>213070643358502:-BACKEND KEY DATA pid=232586748, key=226557416
+server-->>213070643358502:-READY FOR QUERY type=<IDLE>
+213070643358502->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
+213070643358502->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358502->>server:+EXECUTE name='', nb_rows=1
+213070643358502->>server:+SYNC
+server-->>213070643358502:-PARSE COMPLETE
+server-->>213070643358502:-BIND COMPLETE
+server-->>213070643358502:-COMMAND COMPLETE command='SET'
+server-->>213070643358502:-READY FOR QUERY type=<IDLE>
+213070643358502->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
+213070643358502->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358502->>server:+EXECUTE name='', nb_rows=1
+213070643358502->>server:+SYNC
+server-->>213070643358502:-PARSE COMPLETE
+server-->>213070643358502:-BIND COMPLETE
+server-->>213070643358502:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
+server-->>213070643358502:-COMMAND COMPLETE command='SET'
+server-->>213070643358502:-READY FOR QUERY type=<IDLE>
+213070643358502->>server:+PARSE name='', num_params=0, params_type=, query=
+213070643358502->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358502->>server:+DESCRIBE kind='P', name=''
+213070643358502->>server:+EXECUTE name='', nb_rows=1
+213070643358502->>server:+SYNC
+server-->>213070643358502:-PARSE COMPLETE
+server-->>213070643358502:-BIND COMPLETE
+server-->>213070643358502:-NO DATA
+server-->>213070643358502:-EMPTY QUERY RESPONSE
+server-->>213070643358502:-READY FOR QUERY type=<IDLE>
+213070643358502->>server:+PARSE name='', num_params=0, params_type=, query=SELECT VERSION() AS version
+213070643358502->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643358502->>server:+DESCRIBE kind='P', name=''
+213070643358502->>server:+EXECUTE name='', nb_rows=0
+213070643358502->>server:+SYNC
+server-->>213070643358502:-PARSE COMPLETE
+server-->>213070643358502:-BIND COMPLETE
+server-->>213070643358502:-ROW DESCRIPTION: num_fields=1  ---[Field 01]---  name='version'  type=25  type_len=65535  type_mod=4294967295  relid=0  attnum=0  format=0
+server-->>213070643358502:-DATA ROW num_values=1  ---[Value 0001]---  length=27  value='PostgresSQL 13.0 (molecula)'
+server-->>213070643358502:-COMMAND COMPLETE command='SELECT'
+server-->>213070643358502:-READY FOR QUERY type=<IDLE>
+213070643358502->>server:+DISCONNECT
+```
diff --git a/pg/lookerToPostgres.md b/pg/lookerToPostgres.md
new file mode 100644
index 000000000..53ef945be
--- /dev/null
+++ b/pg/lookerToPostgres.md
@@ -0,0 +1,677 @@
+```mermaid
+sequenceDiagram
+
+participant 213070643360888 as c0
+participant 213070643360892 as c1
+participant 213070643360896 as c2
+participant 213070643360900 as c3
+participant 213070643360904 as c4
+participant 213070643360908 as c5
+participant 213070643360912 as c6
+participant 213070643360916 as c7
+participant 213070643360920 as c8
+participant 213070643360924 as c9
+participant 213070643360928 as c10
+participant 213070643360932 as c11
+213070643360888->>server:+SSL REQUEST
+server-->>213070643360888:-SSL BACKEND ANSWER: N
+213070643360888->>server:+STARTUP MESSAGE version: 3  database=trait_store  extra_float_digits=2  TimeZone=GMT  client_encoding=UTF8  user=docker  DateStyle=ISO
+server-->>213070643360888:-AUTHENTIFICATION REQUEST code=5 (MD5 salt='affbc4bf')
+213070643360888->>server:+PASSWORD MESSAGE password=md5de1ae46649b137ee14e14d8fd5fc6cb6
+server-->>213070643360888:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
+server-->>213070643360888:-PARAMETER STATUS name='application_name', value=''
+server-->>213070643360888:-PARAMETER STATUS name='client_encoding', value='UTF8'
+server-->>213070643360888:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
+server-->>213070643360888:-PARAMETER STATUS name='integer_datetimes', value='on'
+server-->>213070643360888:-PARAMETER STATUS name='IntervalStyle', value='postgres'
+server-->>213070643360888:-PARAMETER STATUS name='is_superuser', value='on'
+server-->>213070643360888:-PARAMETER STATUS name='server_encoding', value='UTF8'
+server-->>213070643360888:-PARAMETER STATUS name='server_version', value='13.3 (Debian 13.3-1.pgdg100+1)'
+server-->>213070643360888:-PARAMETER STATUS name='session_authorization', value='docker'
+server-->>213070643360888:-PARAMETER STATUS name='standard_conforming_strings', value='on'
+server-->>213070643360888:-PARAMETER STATUS name='TimeZone', value='GMT'
+server-->>213070643360888:-BACKEND KEY DATA pid=97, key=2120775944
+server-->>213070643360888:-READY FOR QUERY type=<IDLE>
+213070643360888->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
+213070643360888->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360888->>server:+EXECUTE name='', nb_rows=1
+213070643360888->>server:+SYNC
+server-->>213070643360888:-PARSE COMPLETE
+server-->>213070643360888:-BIND COMPLETE
+server-->>213070643360888:-COMMAND COMPLETE command='SET'
+server-->>213070643360888:-READY FOR QUERY type=<IDLE>
+213070643360888->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
+213070643360888->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360888->>server:+EXECUTE name='', nb_rows=1
+213070643360888->>server:+SYNC
+server-->>213070643360888:-PARSE COMPLETE
+server-->>213070643360888:-BIND COMPLETE
+server-->>213070643360888:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
+server-->>213070643360888:-COMMAND COMPLETE command='SET'
+server-->>213070643360888:-READY FOR QUERY type=<IDLE>
+213070643360888->>server:+PARSE name='', num_params=0, params_type=, query=
+213070643360888->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360888->>server:+DESCRIBE kind='P', name=''
+213070643360888->>server:+EXECUTE name='', nb_rows=1
+213070643360888->>server:+SYNC
+server-->>213070643360888:-PARSE COMPLETE
+server-->>213070643360888:-BIND COMPLETE
+server-->>213070643360888:-NO DATA
+server-->>213070643360888:-EMPTY QUERY RESPONSE
+server-->>213070643360888:-READY FOR QUERY type=<IDLE>
+213070643360888->>server:+DISCONNECT
+213070643360892->>server:+SSL REQUEST
+server-->>213070643360892:-SSL BACKEND ANSWER: N
+213070643360892->>server:+STARTUP MESSAGE version: 3  database=trait_store  extra_float_digits=2  TimeZone=GMT  client_encoding=UTF8  user=docker  DateStyle=ISO
+server-->>213070643360892:-AUTHENTIFICATION REQUEST code=5 (MD5 salt='0b76ce86')
+213070643360892->>server:+PASSWORD MESSAGE password=md5b82e4b6283fe5694c0199ca058378bb8
+server-->>213070643360892:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
+server-->>213070643360892:-PARAMETER STATUS name='application_name', value=''
+server-->>213070643360892:-PARAMETER STATUS name='client_encoding', value='UTF8'
+server-->>213070643360892:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
+server-->>213070643360892:-PARAMETER STATUS name='integer_datetimes', value='on'
+server-->>213070643360892:-PARAMETER STATUS name='IntervalStyle', value='postgres'
+server-->>213070643360892:-PARAMETER STATUS name='is_superuser', value='on'
+server-->>213070643360892:-PARAMETER STATUS name='server_encoding', value='UTF8'
+server-->>213070643360892:-PARAMETER STATUS name='server_version', value='13.3 (Debian 13.3-1.pgdg100+1)'
+server-->>213070643360892:-PARAMETER STATUS name='session_authorization', value='docker'
+server-->>213070643360892:-PARAMETER STATUS name='standard_conforming_strings', value='on'
+server-->>213070643360892:-PARAMETER STATUS name='TimeZone', value='GMT'
+server-->>213070643360892:-BACKEND KEY DATA pid=98, key=1329847468
+server-->>213070643360892:-READY FOR QUERY type=<IDLE>
+213070643360892->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
+213070643360892->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360892->>server:+EXECUTE name='', nb_rows=1
+213070643360892->>server:+SYNC
+server-->>213070643360892:-PARSE COMPLETE
+server-->>213070643360892:-BIND COMPLETE
+server-->>213070643360892:-COMMAND COMPLETE command='SET'
+server-->>213070643360892:-READY FOR QUERY type=<IDLE>
+213070643360892->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
+213070643360892->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360892->>server:+EXECUTE name='', nb_rows=1
+213070643360892->>server:+SYNC
+server-->>213070643360892:-PARSE COMPLETE
+server-->>213070643360892:-BIND COMPLETE
+server-->>213070643360892:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
+server-->>213070643360892:-COMMAND COMPLETE command='SET'
+server-->>213070643360892:-READY FOR QUERY type=<IDLE>
+213070643360892->>server:+PARSE name='', num_params=0, params_type=, query=
+213070643360892->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360892->>server:+DESCRIBE kind='P', name=''
+213070643360892->>server:+EXECUTE name='', nb_rows=1
+213070643360892->>server:+SYNC
+server-->>213070643360892:-PARSE COMPLETE
+server-->>213070643360892:-BIND COMPLETE
+server-->>213070643360892:-NO DATA
+server-->>213070643360892:-EMPTY QUERY RESPONSE
+server-->>213070643360892:-READY FOR QUERY type=<IDLE>
+213070643360892->>server:+DISCONNECT
+213070643360896->>server:+SSL REQUEST
+server-->>213070643360896:-SSL BACKEND ANSWER: N
+213070643360896->>server:+STARTUP MESSAGE version: 3  database=trait_store  extra_float_digits=2  TimeZone=GMT  client_encoding=UTF8  user=docker  DateStyle=ISO
+server-->>213070643360896:-AUTHENTIFICATION REQUEST code=5 (MD5 salt='80e88231')
+213070643360896->>server:+PASSWORD MESSAGE password=md59023db05ad8c94976359641cf0ada45a
+server-->>213070643360896:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
+server-->>213070643360896:-PARAMETER STATUS name='application_name', value=''
+server-->>213070643360896:-PARAMETER STATUS name='client_encoding', value='UTF8'
+server-->>213070643360896:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
+server-->>213070643360896:-PARAMETER STATUS name='integer_datetimes', value='on'
+server-->>213070643360896:-PARAMETER STATUS name='IntervalStyle', value='postgres'
+server-->>213070643360896:-PARAMETER STATUS name='is_superuser', value='on'
+server-->>213070643360896:-PARAMETER STATUS name='server_encoding', value='UTF8'
+server-->>213070643360896:-PARAMETER STATUS name='server_version', value='13.3 (Debian 13.3-1.pgdg100+1)'
+server-->>213070643360896:-PARAMETER STATUS name='session_authorization', value='docker'
+server-->>213070643360896:-PARAMETER STATUS name='standard_conforming_strings', value='on'
+server-->>213070643360896:-PARAMETER STATUS name='TimeZone', value='GMT'
+server-->>213070643360896:-BACKEND KEY DATA pid=99, key=1084480878
+server-->>213070643360896:-READY FOR QUERY type=<IDLE>
+213070643360896->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
+213070643360896->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360896->>server:+EXECUTE name='', nb_rows=1
+213070643360896->>server:+SYNC
+server-->>213070643360896:-PARSE COMPLETE
+server-->>213070643360896:-BIND COMPLETE
+server-->>213070643360896:-COMMAND COMPLETE command='SET'
+server-->>213070643360896:-READY FOR QUERY type=<IDLE>
+213070643360896->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
+213070643360896->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360896->>server:+EXECUTE name='', nb_rows=1
+213070643360896->>server:+SYNC
+server-->>213070643360896:-PARSE COMPLETE
+server-->>213070643360896:-BIND COMPLETE
+server-->>213070643360896:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
+server-->>213070643360896:-COMMAND COMPLETE command='SET'
+server-->>213070643360896:-READY FOR QUERY type=<IDLE>
+213070643360896->>server:+PARSE name='', num_params=0, params_type=, query=
+213070643360896->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360896->>server:+DESCRIBE kind='P', name=''
+213070643360896->>server:+EXECUTE name='', nb_rows=1
+213070643360896->>server:+SYNC
+server-->>213070643360896:-PARSE COMPLETE
+server-->>213070643360896:-BIND COMPLETE
+server-->>213070643360896:-NO DATA
+server-->>213070643360896:-EMPTY QUERY RESPONSE
+server-->>213070643360896:-READY FOR QUERY type=<IDLE>
+213070643360896->>server:+DISCONNECT
+213070643360900->>server:+SSL REQUEST
+server-->>213070643360900:-SSL BACKEND ANSWER: N
+213070643360900->>server:+STARTUP MESSAGE version: 3  database=trait_store  extra_float_digits=2  TimeZone=GMT  client_encoding=UTF8  user=docker  DateStyle=ISO
+server-->>213070643360900:-AUTHENTIFICATION REQUEST code=5 (MD5 salt='68a07783')
+213070643360900->>server:+PASSWORD MESSAGE password=md5136c8b8ec47347f93827ec5d7023199c
+server-->>213070643360900:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
+server-->>213070643360900:-PARAMETER STATUS name='application_name', value=''
+server-->>213070643360900:-PARAMETER STATUS name='client_encoding', value='UTF8'
+server-->>213070643360900:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
+server-->>213070643360900:-PARAMETER STATUS name='integer_datetimes', value='on'
+server-->>213070643360900:-PARAMETER STATUS name='IntervalStyle', value='postgres'
+server-->>213070643360900:-PARAMETER STATUS name='is_superuser', value='on'
+server-->>213070643360900:-PARAMETER STATUS name='server_encoding', value='UTF8'
+server-->>213070643360900:-PARAMETER STATUS name='server_version', value='13.3 (Debian 13.3-1.pgdg100+1)'
+server-->>213070643360900:-PARAMETER STATUS name='session_authorization', value='docker'
+server-->>213070643360900:-PARAMETER STATUS name='standard_conforming_strings', value='on'
+server-->>213070643360900:-PARAMETER STATUS name='TimeZone', value='GMT'
+server-->>213070643360900:-BACKEND KEY DATA pid=100, key=594556468
+server-->>213070643360900:-READY FOR QUERY type=<IDLE>
+213070643360900->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
+213070643360900->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360900->>server:+EXECUTE name='', nb_rows=1
+213070643360900->>server:+SYNC
+server-->>213070643360900:-PARSE COMPLETE
+server-->>213070643360900:-BIND COMPLETE
+server-->>213070643360900:-COMMAND COMPLETE command='SET'
+server-->>213070643360900:-READY FOR QUERY type=<IDLE>
+213070643360900->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
+213070643360900->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360900->>server:+EXECUTE name='', nb_rows=1
+213070643360900->>server:+SYNC
+server-->>213070643360900:-PARSE COMPLETE
+server-->>213070643360900:-BIND COMPLETE
+server-->>213070643360900:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
+server-->>213070643360900:-COMMAND COMPLETE command='SET'
+server-->>213070643360900:-READY FOR QUERY type=<IDLE>
+213070643360900->>server:+PARSE name='', num_params=0, params_type=, query=
+213070643360900->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360900->>server:+DESCRIBE kind='P', name=''
+213070643360900->>server:+EXECUTE name='', nb_rows=1
+213070643360900->>server:+SYNC
+server-->>213070643360900:-PARSE COMPLETE
+server-->>213070643360900:-BIND COMPLETE
+server-->>213070643360900:-NO DATA
+server-->>213070643360900:-EMPTY QUERY RESPONSE
+server-->>213070643360900:-READY FOR QUERY type=<IDLE>
+213070643360900->>server:+PARSE name='', num_params=0, params_type=, query=SELECT pg_backend_pid()
+213070643360900->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360900->>server:+DESCRIBE kind='P', name=''
+213070643360900->>server:+EXECUTE name='', nb_rows=0
+213070643360900->>server:+SYNC
+server-->>213070643360900:-PARSE COMPLETE
+server-->>213070643360900:-BIND COMPLETE
+server-->>213070643360900:-ROW DESCRIPTION: num_fields=1  ---[Field 01]---  name='pg_backend_pid'  type=23  type_len=4  type_mod=4294967295  relid=0  attnum=0  format=0
+server-->>213070643360900:-DATA ROW num_values=1  ---[Value 0001]---  length=3  value='100'
+server-->>213070643360900:-COMMAND COMPLETE command='SELECT 1'
+server-->>213070643360900:-READY FOR QUERY type=<IDLE>
+213070643360900->>server:+PARSE name='', num_params=0, params_type=, query=SELECT VERSION() AS version
+213070643360900->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360900->>server:+DESCRIBE kind='P', name=''
+213070643360900->>server:+EXECUTE name='', nb_rows=0
+213070643360900->>server:+SYNC
+server-->>213070643360900:-PARSE COMPLETE
+server-->>213070643360900:-BIND COMPLETE
+server-->>213070643360900:-ROW DESCRIPTION: num_fields=1  ---[Field 01]---  name='version'  type=25  type_len=65535  type_mod=4294967295  relid=0  attnum=0  format=0
+server-->>213070643360900:-DATA ROW num_values=1  ---[Value 0001]---  length=112  value='PostgreSQL 13.3 (Debian 13.3-1.pgdg100+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 8.3.0-6) 8.3.0, 64-bit'
+server-->>213070643360900:-COMMAND COMPLETE command='SELECT 1'
+server-->>213070643360900:-READY FOR QUERY type=<IDLE>
+213070643360900->>server:+PARSE name='', num_params=0, params_type=, query=        SELECT COUNT(*)        FROM pg_type AS t0,             pg_aggregate AS t1,             pg_settings AS t2,             pg_settings AS t3
+213070643360900->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360900->>server:+DESCRIBE kind='P', name=''
+213070643360900->>server:+EXECUTE name='', nb_rows=0
+213070643360900->>server:+SYNC
+213070643360904->>server:+SSL REQUEST
+server-->>213070643360904:-SSL BACKEND ANSWER: N
+213070643360904->>server:+STARTUP MESSAGE version: 3  database=trait_store  extra_float_digits=2  TimeZone=GMT  client_encoding=UTF8  user=docker  DateStyle=ISO
+server-->>213070643360904:-AUTHENTIFICATION REQUEST code=5 (MD5 salt='a03ee692')
+213070643360904->>server:+PASSWORD MESSAGE password=md5cfedb52ae0cafa87a4f1066df3ba6802
+server-->>213070643360904:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
+server-->>213070643360904:-PARAMETER STATUS name='application_name', value=''
+server-->>213070643360904:-PARAMETER STATUS name='client_encoding', value='UTF8'
+server-->>213070643360904:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
+server-->>213070643360904:-PARAMETER STATUS name='integer_datetimes', value='on'
+server-->>213070643360904:-PARAMETER STATUS name='IntervalStyle', value='postgres'
+server-->>213070643360904:-PARAMETER STATUS name='is_superuser', value='on'
+server-->>213070643360904:-PARAMETER STATUS name='server_encoding', value='UTF8'
+server-->>213070643360904:-PARAMETER STATUS name='server_version', value='13.3 (Debian 13.3-1.pgdg100+1)'
+server-->>213070643360904:-PARAMETER STATUS name='session_authorization', value='docker'
+server-->>213070643360904:-PARAMETER STATUS name='standard_conforming_strings', value='on'
+server-->>213070643360904:-PARAMETER STATUS name='TimeZone', value='GMT'
+server-->>213070643360904:-BACKEND KEY DATA pid=101, key=2854304053
+server-->>213070643360904:-READY FOR QUERY type=<IDLE>
+213070643360904->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
+213070643360904->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360904->>server:+EXECUTE name='', nb_rows=1
+213070643360904->>server:+SYNC
+server-->>213070643360904:-PARSE COMPLETE
+server-->>213070643360904:-BIND COMPLETE
+server-->>213070643360904:-COMMAND COMPLETE command='SET'
+server-->>213070643360904:-READY FOR QUERY type=<IDLE>
+213070643360904->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
+213070643360904->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360904->>server:+EXECUTE name='', nb_rows=1
+213070643360904->>server:+SYNC
+server-->>213070643360904:-PARSE COMPLETE
+server-->>213070643360904:-BIND COMPLETE
+server-->>213070643360904:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
+server-->>213070643360904:-COMMAND COMPLETE command='SET'
+server-->>213070643360904:-READY FOR QUERY type=<IDLE>
+213070643360904->>server:+PARSE name='', num_params=0, params_type=, query=
+213070643360904->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360904->>server:+DESCRIBE kind='P', name=''
+213070643360904->>server:+EXECUTE name='', nb_rows=1
+213070643360904->>server:+SYNC
+server-->>213070643360904:-PARSE COMPLETE
+server-->>213070643360904:-BIND COMPLETE
+server-->>213070643360904:-NO DATA
+server-->>213070643360904:-EMPTY QUERY RESPONSE
+server-->>213070643360904:-READY FOR QUERY type=<IDLE>
+213070643360904->>server:+DISCONNECT
+213070643360908->>server:+SSL REQUEST
+server-->>213070643360908:-SSL BACKEND ANSWER: N
+213070643360908->>server:+STARTUP MESSAGE version: 3  database=trait_store  extra_float_digits=2  TimeZone=GMT  client_encoding=UTF8  user=docker  DateStyle=ISO
+server-->>213070643360908:-AUTHENTIFICATION REQUEST code=5 (MD5 salt='be3afb9d')
+213070643360908->>server:+PASSWORD MESSAGE password=md57b5cfa30f89fc7733c814addcd548c03
+server-->>213070643360908:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
+server-->>213070643360908:-PARAMETER STATUS name='application_name', value=''
+server-->>213070643360908:-PARAMETER STATUS name='client_encoding', value='UTF8'
+server-->>213070643360908:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
+server-->>213070643360908:-PARAMETER STATUS name='integer_datetimes', value='on'
+server-->>213070643360908:-PARAMETER STATUS name='IntervalStyle', value='postgres'
+server-->>213070643360908:-PARAMETER STATUS name='is_superuser', value='on'
+server-->>213070643360908:-PARAMETER STATUS name='server_encoding', value='UTF8'
+server-->>213070643360908:-PARAMETER STATUS name='server_version', value='13.3 (Debian 13.3-1.pgdg100+1)'
+server-->>213070643360908:-PARAMETER STATUS name='session_authorization', value='docker'
+server-->>213070643360908:-PARAMETER STATUS name='standard_conforming_strings', value='on'
+server-->>213070643360908:-PARAMETER STATUS name='TimeZone', value='GMT'
+server-->>213070643360908:-BACKEND KEY DATA pid=102, key=3089396074
+server-->>213070643360908:-READY FOR QUERY type=<IDLE>
+213070643360908->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
+213070643360908->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360908->>server:+EXECUTE name='', nb_rows=1
+213070643360908->>server:+SYNC
+server-->>213070643360908:-PARSE COMPLETE
+server-->>213070643360908:-BIND COMPLETE
+server-->>213070643360908:-COMMAND COMPLETE command='SET'
+server-->>213070643360908:-READY FOR QUERY type=<IDLE>
+213070643360908->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
+213070643360908->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360908->>server:+EXECUTE name='', nb_rows=1
+213070643360908->>server:+SYNC
+server-->>213070643360908:-PARSE COMPLETE
+server-->>213070643360908:-BIND COMPLETE
+server-->>213070643360908:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
+server-->>213070643360908:-COMMAND COMPLETE command='SET'
+server-->>213070643360908:-READY FOR QUERY type=<IDLE>
+213070643360908->>server:+PARSE name='', num_params=0, params_type=, query=
+213070643360908->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360908->>server:+DESCRIBE kind='P', name=''
+213070643360908->>server:+EXECUTE name='', nb_rows=1
+213070643360908->>server:+SYNC
+server-->>213070643360908:-PARSE COMPLETE
+server-->>213070643360908:-BIND COMPLETE
+server-->>213070643360908:-NO DATA
+server-->>213070643360908:-EMPTY QUERY RESPONSE
+server-->>213070643360908:-READY FOR QUERY type=<IDLE>
+213070643360908->>server:+PARSE name='', num_params=0, params_type=, query=SELECT VERSION() AS version
+213070643360908->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360908->>server:+DESCRIBE kind='P', name=''
+213070643360908->>server:+EXECUTE name='', nb_rows=0
+213070643360908->>server:+SYNC
+server-->>213070643360908:-PARSE COMPLETE
+server-->>213070643360908:-BIND COMPLETE
+server-->>213070643360908:-ROW DESCRIPTION: num_fields=1  ---[Field 01]---  name='version'  type=25  type_len=65535  type_mod=4294967295  relid=0  attnum=0  format=0
+server-->>213070643360908:-DATA ROW num_values=1  ---[Value 0001]---  length=112  value='PostgreSQL 13.3 (Debian 13.3-1.pgdg100+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 8.3.0-6) 8.3.0, 64-bit'
+server-->>213070643360908:-COMMAND COMPLETE command='SELECT 1'
+server-->>213070643360908:-READY FOR QUERY type=<IDLE>
+213070643360908->>server:+PARSE name='', num_params=0, params_type=, query=
+213070643360908->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360908->>server:+DESCRIBE kind='P', name=''
+213070643360908->>server:+EXECUTE name='', nb_rows=1
+213070643360908->>server:+SYNC
+server-->>213070643360908:-PARSE COMPLETE
+server-->>213070643360908:-BIND COMPLETE
+server-->>213070643360908:-NO DATA
+server-->>213070643360908:-EMPTY QUERY RESPONSE
+server-->>213070643360908:-READY FOR QUERY type=<IDLE>
+213070643360908->>server:+PARSE name='', num_params=0, params_type=, query=          SELECT pid as id,            query as stmt,            EXTRACT(seconds from query_start - NOW()) as elapsed_time          FROM pg_stat_activity          WHERE usename='docker'
+213070643360908->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360908->>server:+DESCRIBE kind='P', name=''
+213070643360908->>server:+EXECUTE name='', nb_rows=0
+213070643360908->>server:+SYNC
+server-->>213070643360908:-PARSE COMPLETE
+server-->>213070643360908:-BIND COMPLETE
+server-->>213070643360908:-ROW DESCRIPTION: num_fields=3  ---[Field 01]---  name='id'  type=23  type_len=4  type_mod=4294967295  relid=12250  attnum=3  format=0  ---[Field 02]---  name='stmt'  type=25  type_len=65535  type_mod=4294967295  relid=12250  attnum=20  format=0  ---[Field 03]---  name='elapsed_time'  type=701  type_len=8  type_mod=4294967295  relid=0  attnum=0  format=0
+server-->>213070643360908:-DATA ROW num_values=3  ---[Value 0001]---  length=2  value='81'  ---[Value 0002]---  length=0  value=''  ---[Value 0003]---  length=-1  value=NULL
+server-->>213070643360908:-DATA ROW num_values=3  ---[Value 0001]---  length=3  value='100'  ---[Value 0002]---  length=148  value='        SELECT COUNT(*).        FROM pg_type AS t0,.             pg_aggregate AS t1,.             pg_settings AS t2,.             pg_settings AS t3.'  ---[Value 0003]---  length=9  value='-1.155449'
+server-->>213070643360908:-DATA ROW num_values=3  ---[Value 0001]---  length=3  value='102'  ---[Value 0002]---  length=190  value='          SELECT pid as id,.            query as stmt,.            EXTRACT(seconds from query_start - NOW()) as elapsed_time.          FROM pg_stat_activity.          WHERE usename='docker'.'  ---[Value 0003]---  length=8  value='0.002903'
+server-->>213070643360908:-COMMAND COMPLETE command='SELECT 3'
+server-->>213070643360908:-READY FOR QUERY type=<IDLE>
+213070643360912->>server:+SSL REQUEST
+server-->>213070643360912:-SSL BACKEND ANSWER: N
+213070643360912->>server:+STARTUP MESSAGE version: 3  database=trait_store  extra_float_digits=2  TimeZone=GMT  client_encoding=UTF8  user=docker  DateStyle=ISO
+server-->>213070643360912:-AUTHENTIFICATION REQUEST code=5 (MD5 salt='14444412')
+213070643360912->>server:+PASSWORD MESSAGE password=md59cef18c88b7988d7ac2f215fc1569c62
+server-->>213070643360912:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
+server-->>213070643360912:-PARAMETER STATUS name='application_name', value=''
+server-->>213070643360912:-PARAMETER STATUS name='client_encoding', value='UTF8'
+server-->>213070643360912:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
+server-->>213070643360912:-PARAMETER STATUS name='integer_datetimes', value='on'
+server-->>213070643360912:-PARAMETER STATUS name='IntervalStyle', value='postgres'
+server-->>213070643360912:-PARAMETER STATUS name='is_superuser', value='on'
+server-->>213070643360912:-PARAMETER STATUS name='server_encoding', value='UTF8'
+server-->>213070643360912:-PARAMETER STATUS name='server_version', value='13.3 (Debian 13.3-1.pgdg100+1)'
+server-->>213070643360912:-PARAMETER STATUS name='session_authorization', value='docker'
+server-->>213070643360912:-PARAMETER STATUS name='standard_conforming_strings', value='on'
+server-->>213070643360912:-PARAMETER STATUS name='TimeZone', value='GMT'
+server-->>213070643360912:-BACKEND KEY DATA pid=103, key=1911102642
+server-->>213070643360912:-READY FOR QUERY type=<IDLE>
+213070643360912->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
+213070643360912->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360912->>server:+EXECUTE name='', nb_rows=1
+213070643360912->>server:+SYNC
+server-->>213070643360912:-PARSE COMPLETE
+server-->>213070643360912:-BIND COMPLETE
+server-->>213070643360912:-COMMAND COMPLETE command='SET'
+server-->>213070643360912:-READY FOR QUERY type=<IDLE>
+213070643360912->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
+213070643360912->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360912->>server:+EXECUTE name='', nb_rows=1
+213070643360912->>server:+SYNC
+server-->>213070643360912:-PARSE COMPLETE
+server-->>213070643360912:-BIND COMPLETE
+server-->>213070643360912:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
+server-->>213070643360912:-COMMAND COMPLETE command='SET'
+server-->>213070643360912:-READY FOR QUERY type=<IDLE>
+213070643360912->>server:+PARSE name='', num_params=0, params_type=, query=
+213070643360912->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360912->>server:+DESCRIBE kind='P', name=''
+213070643360912->>server:+EXECUTE name='', nb_rows=1
+213070643360912->>server:+SYNC
+server-->>213070643360912:-PARSE COMPLETE
+server-->>213070643360912:-BIND COMPLETE
+server-->>213070643360912:-NO DATA
+server-->>213070643360912:-EMPTY QUERY RESPONSE
+server-->>213070643360912:-READY FOR QUERY type=<IDLE>
+213070643360912->>server:+DISCONNECT
+213070643360916->>server:+SSL REQUEST
+server-->>213070643360916:-SSL BACKEND ANSWER: N
+213070643360916->>server:+STARTUP MESSAGE version: 3  database=trait_store  extra_float_digits=2  TimeZone=GMT  client_encoding=UTF8  user=docker  DateStyle=ISO
+server-->>213070643360916:-AUTHENTIFICATION REQUEST code=5 (MD5 salt='9eb7546f')
+213070643360916->>server:+PASSWORD MESSAGE password=md5c7655226306ab14fa17e44dba876a348
+server-->>213070643360916:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
+server-->>213070643360916:-PARAMETER STATUS name='application_name', value=''
+server-->>213070643360916:-PARAMETER STATUS name='client_encoding', value='UTF8'
+server-->>213070643360916:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
+server-->>213070643360916:-PARAMETER STATUS name='integer_datetimes', value='on'
+server-->>213070643360916:-PARAMETER STATUS name='IntervalStyle', value='postgres'
+server-->>213070643360916:-PARAMETER STATUS name='is_superuser', value='on'
+server-->>213070643360916:-PARAMETER STATUS name='server_encoding', value='UTF8'
+server-->>213070643360916:-PARAMETER STATUS name='server_version', value='13.3 (Debian 13.3-1.pgdg100+1)'
+server-->>213070643360916:-PARAMETER STATUS name='session_authorization', value='docker'
+server-->>213070643360916:-PARAMETER STATUS name='standard_conforming_strings', value='on'
+server-->>213070643360916:-PARAMETER STATUS name='TimeZone', value='GMT'
+server-->>213070643360916:-BACKEND KEY DATA pid=104, key=927987783
+server-->>213070643360916:-READY FOR QUERY type=<IDLE>
+213070643360916->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
+213070643360916->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360916->>server:+EXECUTE name='', nb_rows=1
+213070643360916->>server:+SYNC
+server-->>213070643360916:-PARSE COMPLETE
+server-->>213070643360916:-BIND COMPLETE
+server-->>213070643360916:-COMMAND COMPLETE command='SET'
+server-->>213070643360916:-READY FOR QUERY type=<IDLE>
+213070643360916->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
+213070643360916->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360916->>server:+EXECUTE name='', nb_rows=1
+213070643360916->>server:+SYNC
+server-->>213070643360916:-PARSE COMPLETE
+server-->>213070643360916:-BIND COMPLETE
+server-->>213070643360916:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
+server-->>213070643360916:-COMMAND COMPLETE command='SET'
+server-->>213070643360916:-READY FOR QUERY type=<IDLE>
+213070643360916->>server:+PARSE name='', num_params=0, params_type=, query=
+213070643360916->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360916->>server:+DESCRIBE kind='P', name=''
+213070643360916->>server:+EXECUTE name='', nb_rows=1
+213070643360916->>server:+SYNC
+server-->>213070643360916:-PARSE COMPLETE
+server-->>213070643360916:-BIND COMPLETE
+server-->>213070643360916:-NO DATA
+server-->>213070643360916:-EMPTY QUERY RESPONSE
+server-->>213070643360916:-READY FOR QUERY type=<IDLE>
+213070643360916->>server:+PARSE name='', num_params=0, params_type=, query=select pg_terminate_backend(100)
+213070643360916->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360916->>server:+DESCRIBE kind='P', name=''
+213070643360916->>server:+EXECUTE name='', nb_rows=0
+213070643360916->>server:+SYNC
+server-->>213070643360916:-PARSE COMPLETE
+server-->>213070643360916:-BIND COMPLETE
+server-->>213070643360916:-ROW DESCRIPTION: num_fields=1  ---[Field 01]---  name='pg_terminate_backend'  type=16  type_len=1  type_mod=4294967295  relid=0  attnum=0  format=0
+server-->>213070643360916:-DATA ROW num_values=1  ---[Value 0001]---  length=1  value='t'
+server-->>213070643360916:-COMMAND COMPLETE command='SELECT 1'
+server-->>213070643360916:-READY FOR QUERY type=<IDLE>
+server-->>213070643360900:-PARSE COMPLETE
+server-->>213070643360900:-BIND COMPLETE
+server-->>213070643360900:-ROW DESCRIPTION: num_fields=1  ---[Field 01]---  name='count'  type=20  type_len=8  type_mod=4294967295  relid=0  attnum=0  format=0
+server-->>213070643360900:-ERROR RESPONSE  File: 'postgres.c'  Severity: 'FATAL'  Message: 'terminating connection due to administrator command'  Code: '57P01'  Routine: 'ProcessInterrupts'  Line: '3090'
+213070643360908->>server:+DISCONNECT
+213070643360916->>server:+DISCONNECT
+213070643360920->>server:+SSL REQUEST
+server-->>213070643360920:-SSL BACKEND ANSWER: N
+213070643360920->>server:+STARTUP MESSAGE version: 3  database=trait_store  extra_float_digits=2  TimeZone=GMT  client_encoding=UTF8  user=docker  DateStyle=ISO
+server-->>213070643360920:-AUTHENTIFICATION REQUEST code=5 (MD5 salt='1ad4dcf7')
+213070643360920->>server:+PASSWORD MESSAGE password=md554de511c5219f67a8bbe92da445fa5a2
+server-->>213070643360920:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
+server-->>213070643360920:-PARAMETER STATUS name='application_name', value=''
+server-->>213070643360920:-PARAMETER STATUS name='client_encoding', value='UTF8'
+server-->>213070643360920:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
+server-->>213070643360920:-PARAMETER STATUS name='integer_datetimes', value='on'
+server-->>213070643360920:-PARAMETER STATUS name='IntervalStyle', value='postgres'
+server-->>213070643360920:-PARAMETER STATUS name='is_superuser', value='on'
+server-->>213070643360920:-PARAMETER STATUS name='server_encoding', value='UTF8'
+server-->>213070643360920:-PARAMETER STATUS name='server_version', value='13.3 (Debian 13.3-1.pgdg100+1)'
+server-->>213070643360920:-PARAMETER STATUS name='session_authorization', value='docker'
+server-->>213070643360920:-PARAMETER STATUS name='standard_conforming_strings', value='on'
+server-->>213070643360920:-PARAMETER STATUS name='TimeZone', value='GMT'
+server-->>213070643360920:-BACKEND KEY DATA pid=105, key=3630332440
+server-->>213070643360920:-READY FOR QUERY type=<IDLE>
+213070643360920->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
+213070643360920->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360920->>server:+EXECUTE name='', nb_rows=1
+213070643360920->>server:+SYNC
+server-->>213070643360920:-PARSE COMPLETE
+server-->>213070643360920:-BIND COMPLETE
+server-->>213070643360920:-COMMAND COMPLETE command='SET'
+server-->>213070643360920:-READY FOR QUERY type=<IDLE>
+213070643360920->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
+213070643360920->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360920->>server:+EXECUTE name='', nb_rows=1
+213070643360920->>server:+SYNC
+server-->>213070643360920:-PARSE COMPLETE
+server-->>213070643360920:-BIND COMPLETE
+server-->>213070643360920:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
+server-->>213070643360920:-COMMAND COMPLETE command='SET'
+server-->>213070643360920:-READY FOR QUERY type=<IDLE>
+213070643360920->>server:+PARSE name='', num_params=0, params_type=, query=
+213070643360920->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360920->>server:+DESCRIBE kind='P', name=''
+213070643360920->>server:+EXECUTE name='', nb_rows=1
+213070643360920->>server:+SYNC
+server-->>213070643360920:-PARSE COMPLETE
+server-->>213070643360920:-BIND COMPLETE
+server-->>213070643360920:-NO DATA
+server-->>213070643360920:-EMPTY QUERY RESPONSE
+server-->>213070643360920:-READY FOR QUERY type=<IDLE>
+213070643360920->>server:+DISCONNECT
+213070643360924->>server:+SSL REQUEST
+server-->>213070643360924:-SSL BACKEND ANSWER: N
+213070643360924->>server:+STARTUP MESSAGE version: 3  database=trait_store  extra_float_digits=2  TimeZone=GMT  client_encoding=UTF8  user=docker  DateStyle=ISO
+server-->>213070643360924:-AUTHENTIFICATION REQUEST code=5 (MD5 salt='494c352f')
+213070643360924->>server:+PASSWORD MESSAGE password=md5919fc9ed056904fa0d8e1dd00625a556
+server-->>213070643360924:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
+server-->>213070643360924:-PARAMETER STATUS name='application_name', value=''
+server-->>213070643360924:-PARAMETER STATUS name='client_encoding', value='UTF8'
+server-->>213070643360924:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
+server-->>213070643360924:-PARAMETER STATUS name='integer_datetimes', value='on'
+server-->>213070643360924:-PARAMETER STATUS name='IntervalStyle', value='postgres'
+server-->>213070643360924:-PARAMETER STATUS name='is_superuser', value='on'
+server-->>213070643360924:-PARAMETER STATUS name='server_encoding', value='UTF8'
+server-->>213070643360924:-PARAMETER STATUS name='server_version', value='13.3 (Debian 13.3-1.pgdg100+1)'
+server-->>213070643360924:-PARAMETER STATUS name='session_authorization', value='docker'
+server-->>213070643360924:-PARAMETER STATUS name='standard_conforming_strings', value='on'
+server-->>213070643360924:-PARAMETER STATUS name='TimeZone', value='GMT'
+server-->>213070643360924:-BACKEND KEY DATA pid=106, key=3364308023
+server-->>213070643360924:-READY FOR QUERY type=<IDLE>
+213070643360924->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
+213070643360924->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360924->>server:+EXECUTE name='', nb_rows=1
+213070643360924->>server:+SYNC
+server-->>213070643360924:-PARSE COMPLETE
+server-->>213070643360924:-BIND COMPLETE
+server-->>213070643360924:-COMMAND COMPLETE command='SET'
+server-->>213070643360924:-READY FOR QUERY type=<IDLE>
+213070643360924->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
+213070643360924->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360924->>server:+EXECUTE name='', nb_rows=1
+213070643360924->>server:+SYNC
+server-->>213070643360924:-PARSE COMPLETE
+server-->>213070643360924:-BIND COMPLETE
+server-->>213070643360924:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
+server-->>213070643360924:-COMMAND COMPLETE command='SET'
+server-->>213070643360924:-READY FOR QUERY type=<IDLE>
+213070643360924->>server:+PARSE name='', num_params=0, params_type=, query=
+213070643360924->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360924->>server:+DESCRIBE kind='P', name=''
+213070643360924->>server:+EXECUTE name='', nb_rows=1
+213070643360924->>server:+SYNC
+server-->>213070643360924:-PARSE COMPLETE
+server-->>213070643360924:-BIND COMPLETE
+server-->>213070643360924:-NO DATA
+server-->>213070643360924:-EMPTY QUERY RESPONSE
+server-->>213070643360924:-READY FOR QUERY type=<IDLE>
+213070643360924->>server:+PARSE name='', num_params=0, params_type=, query=SELECT 1 
+213070643360924->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360924->>server:+DESCRIBE kind='P', name=''
+213070643360924->>server:+EXECUTE name='', nb_rows=0
+213070643360924->>server:+SYNC
+server-->>213070643360924:-PARSE COMPLETE
+server-->>213070643360924:-BIND COMPLETE
+server-->>213070643360924:-ROW DESCRIPTION: num_fields=1  ---[Field 01]---  name='?column?'  type=23  type_len=4  type_mod=4294967295  relid=0  attnum=0  format=0
+server-->>213070643360924:-DATA ROW num_values=1  ---[Value 0001]---  length=1  value='1'
+server-->>213070643360924:-COMMAND COMPLETE command='SELECT 1'
+server-->>213070643360924:-READY FOR QUERY type=<IDLE>
+213070643360924->>server:+DISCONNECT
+213070643360928->>server:+SSL REQUEST
+server-->>213070643360928:-SSL BACKEND ANSWER: N
+213070643360928->>server:+STARTUP MESSAGE version: 3  database=trait_store  extra_float_digits=2  TimeZone=GMT  client_encoding=UTF8  user=docker  DateStyle=ISO
+server-->>213070643360928:-AUTHENTIFICATION REQUEST code=5 (MD5 salt='4b2173db')
+213070643360928->>server:+PASSWORD MESSAGE password=md53977c39c5c7cdef7f80e74b256a8ce25
+server-->>213070643360928:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
+server-->>213070643360928:-PARAMETER STATUS name='application_name', value=''
+server-->>213070643360928:-PARAMETER STATUS name='client_encoding', value='UTF8'
+server-->>213070643360928:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
+server-->>213070643360928:-PARAMETER STATUS name='integer_datetimes', value='on'
+server-->>213070643360928:-PARAMETER STATUS name='IntervalStyle', value='postgres'
+server-->>213070643360928:-PARAMETER STATUS name='is_superuser', value='on'
+server-->>213070643360928:-PARAMETER STATUS name='server_encoding', value='UTF8'
+server-->>213070643360928:-PARAMETER STATUS name='server_version', value='13.3 (Debian 13.3-1.pgdg100+1)'
+server-->>213070643360928:-PARAMETER STATUS name='session_authorization', value='docker'
+server-->>213070643360928:-PARAMETER STATUS name='standard_conforming_strings', value='on'
+server-->>213070643360928:-PARAMETER STATUS name='TimeZone', value='GMT'
+server-->>213070643360928:-BACKEND KEY DATA pid=107, key=1988416582
+server-->>213070643360928:-READY FOR QUERY type=<IDLE>
+213070643360928->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
+213070643360928->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360928->>server:+EXECUTE name='', nb_rows=1
+213070643360928->>server:+SYNC
+server-->>213070643360928:-PARSE COMPLETE
+server-->>213070643360928:-BIND COMPLETE
+server-->>213070643360928:-COMMAND COMPLETE command='SET'
+server-->>213070643360928:-READY FOR QUERY type=<IDLE>
+213070643360928->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
+213070643360928->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360928->>server:+EXECUTE name='', nb_rows=1
+213070643360928->>server:+SYNC
+server-->>213070643360928:-PARSE COMPLETE
+server-->>213070643360928:-BIND COMPLETE
+server-->>213070643360928:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
+server-->>213070643360928:-COMMAND COMPLETE command='SET'
+server-->>213070643360928:-READY FOR QUERY type=<IDLE>
+213070643360928->>server:+PARSE name='', num_params=0, params_type=, query=
+213070643360928->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360928->>server:+DESCRIBE kind='P', name=''
+213070643360928->>server:+EXECUTE name='', nb_rows=1
+213070643360928->>server:+SYNC
+server-->>213070643360928:-PARSE COMPLETE
+server-->>213070643360928:-BIND COMPLETE
+server-->>213070643360928:-NO DATA
+server-->>213070643360928:-EMPTY QUERY RESPONSE
+server-->>213070643360928:-READY FOR QUERY type=<IDLE>
+213070643360928->>server:+DISCONNECT
+213070643360932->>server:+SSL REQUEST
+server-->>213070643360932:-SSL BACKEND ANSWER: N
+213070643360932->>server:+STARTUP MESSAGE version: 3  database=trait_store  extra_float_digits=2  TimeZone=GMT  client_encoding=UTF8  user=docker  DateStyle=ISO
+server-->>213070643360932:-AUTHENTIFICATION REQUEST code=5 (MD5 salt='f6d0f51d')
+213070643360932->>server:+PASSWORD MESSAGE password=md5b5b322fe9bcc7c242c6b44cc8a7898e8
+server-->>213070643360932:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
+server-->>213070643360932:-PARAMETER STATUS name='application_name', value=''
+server-->>213070643360932:-PARAMETER STATUS name='client_encoding', value='UTF8'
+server-->>213070643360932:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
+server-->>213070643360932:-PARAMETER STATUS name='integer_datetimes', value='on'
+server-->>213070643360932:-PARAMETER STATUS name='IntervalStyle', value='postgres'
+server-->>213070643360932:-PARAMETER STATUS name='is_superuser', value='on'
+server-->>213070643360932:-PARAMETER STATUS name='server_encoding', value='UTF8'
+server-->>213070643360932:-PARAMETER STATUS name='server_version', value='13.3 (Debian 13.3-1.pgdg100+1)'
+server-->>213070643360932:-PARAMETER STATUS name='session_authorization', value='docker'
+server-->>213070643360932:-PARAMETER STATUS name='standard_conforming_strings', value='on'
+server-->>213070643360932:-PARAMETER STATUS name='TimeZone', value='GMT'
+server-->>213070643360932:-BACKEND KEY DATA pid=108, key=997991029
+server-->>213070643360932:-READY FOR QUERY type=<IDLE>
+213070643360932->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
+213070643360932->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360932->>server:+EXECUTE name='', nb_rows=1
+213070643360932->>server:+SYNC
+server-->>213070643360932:-PARSE COMPLETE
+server-->>213070643360932:-BIND COMPLETE
+server-->>213070643360932:-COMMAND COMPLETE command='SET'
+server-->>213070643360932:-READY FOR QUERY type=<IDLE>
+213070643360932->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
+213070643360932->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360932->>server:+EXECUTE name='', nb_rows=1
+213070643360932->>server:+SYNC
+server-->>213070643360932:-PARSE COMPLETE
+server-->>213070643360932:-BIND COMPLETE
+server-->>213070643360932:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
+server-->>213070643360932:-COMMAND COMPLETE command='SET'
+server-->>213070643360932:-READY FOR QUERY type=<IDLE>
+213070643360932->>server:+PARSE name='', num_params=0, params_type=, query=
+213070643360932->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360932->>server:+DESCRIBE kind='P', name=''
+213070643360932->>server:+EXECUTE name='', nb_rows=1
+213070643360932->>server:+SYNC
+server-->>213070643360932:-PARSE COMPLETE
+server-->>213070643360932:-BIND COMPLETE
+server-->>213070643360932:-NO DATA
+server-->>213070643360932:-EMPTY QUERY RESPONSE
+server-->>213070643360932:-READY FOR QUERY type=<IDLE>
+213070643360932->>server:+PARSE name='', num_params=0, params_type=, query=SELECT VERSION() AS version
+213070643360932->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
+213070643360932->>server:+DESCRIBE kind='P', name=''
+213070643360932->>server:+EXECUTE name='', nb_rows=0
+213070643360932->>server:+SYNC
+server-->>213070643360932:-PARSE COMPLETE
+server-->>213070643360932:-BIND COMPLETE
+server-->>213070643360932:-ROW DESCRIPTION: num_fields=1  ---[Field 01]---  name='version'  type=25  type_len=65535  type_mod=4294967295  relid=0  attnum=0  format=0
+server-->>213070643360932:-DATA ROW num_values=1  ---[Value 0001]---  length=112  value='PostgreSQL 13.3 (Debian 13.3-1.pgdg100+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 8.3.0-6) 8.3.0, 64-bit'
+server-->>213070643360932:-COMMAND COMPLETE command='SELECT 1'
+server-->>213070643360932:-READY FOR QUERY type=<IDLE>
+213070643360932->>server:+DISCONNECT
+```
diff --git a/pg/message/io.go b/pg/message/io.go
index e09150b9d..ec2321675 100644
--- a/pg/message/io.go
+++ b/pg/message/io.go
@@ -18,6 +18,7 @@ import (
 	"bufio"
 	"encoding/binary"
 	"errors"
+	"fmt"
 	"io"
 )
 
@@ -92,6 +93,7 @@ type WireWriter struct {
 
 // WriteMessage writes a message onto the wire.
 func (w *WireWriter) WriteMessage(message Message) error {
+	fmt.Printf("SendToClient %c (%d)\n", message.Type, len(message.Data))
 	if uint(len(message.Data))+4 >= 1<<31 {
 		return ErrMessageTooBig
 	}
diff --git a/pg/message/message.go b/pg/message/message.go
index caf8214d3..5d6f67014 100644
--- a/pg/message/message.go
+++ b/pg/message/message.go
@@ -17,6 +17,7 @@ package message
 import (
 	"bytes"
 	"encoding/binary"
+	"fmt"
 )
 
 // Type is a byte indicating the type of a Postgres message.
@@ -29,11 +30,11 @@ const (
 	// TypeReadyForQuery is a message used to indicate that the server is ready for another query.
 	TypeReadyForQuery Type = 'Z'
 
-	// TypeCommandComplete is a message used to indicate that a query has completed.
+	// TypeCommandComplete is a message used to indicate that a query has completed. Backend
 	TypeCommandComplete Type = 'C'
 
-	// TypeError is an error message.
-	TypeError Type = 'E'
+	// TypeClos is a message used to indicate that a query has completed. Frontend
+	TypeClose Type = 'C'
 
 	// TypeRowDescription is a message indicating the column types of the result rows from a query.
 	TypeRowDescription Type = 'T'
@@ -52,6 +53,20 @@ const (
 
 	// TypeBackendKeyData contains a cancellation key for the client to use later.
 	TypeBackendKeyData Type = 'K'
+
+	TypeParse         Type = 'P'
+	TypeParseComplete Type = '1'
+
+	TypeBind         Type = 'B'
+	TypeBindComplete Type = '2'
+
+	TypeExecute            Type = 'E' // Frontend TODO(TWG)
+	TypeError              Type = 'E' // Backend   SOMETHING NOT RIGHT HERE
+	TypeSync               Type = 'S' // Frontend
+	TypeParameterStatus    Type = 'S' // Backend
+	TypeDescribe           Type = 'D' // Frontend
+	TypeNoData             Type = 'n' // backend
+	TypeEmptyQueryResponse Type = 'I' // backend
 )
 
 // AuthenticationOK is a message indicating that authentication has completed.
@@ -59,6 +74,22 @@ var AuthenticationOK = Message{
 	Type: TypeAuthentication,
 	Data: []byte{0, 0, 0, 0},
 }
+var ParseOK = Message{
+	Type: TypeParseComplete,
+	Data: []byte{},
+}
+var BindComplete = Message{
+	Type: TypeBindComplete,
+	Data: []byte{},
+}
+var NoData = Message{
+	Type: TypeNoData,
+	Data: []byte{},
+}
+var EmptyQueryResponse = Message{
+	Type: TypeEmptyQueryResponse,
+	Data: []byte{},
+}
 
 // Message is a Postgres message value.
 type Message struct {
@@ -66,6 +97,49 @@ type Message struct {
 	Data []byte
 }
 
+//debug tools
+func viewString(b []byte) string {
+	r := []rune(string(b))
+	for i := range r {
+		if r[i] < 32 || r[i] > 126 {
+			r[i] = '.'
+		}
+	}
+	return string(r)
+}
+func min(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}
+func (m *Message) Dump(prefix string) {
+	n := len(m.Data)
+	rowcount := 0
+	stop := (n / 8) * 8
+	k := 0
+	fmt.Printf("\n %s type: '%c'\n", prefix, m.Type)
+	for i := 0; i <= stop; i += 8 {
+		k++
+		if i+8 < n {
+			rowcount = 8
+		} else {
+			rowcount = min(k*8, n) % 8
+		}
+
+		fmt.Printf("pos %02d  hex:  ", i)
+		for j := 0; j < rowcount; j++ {
+			fmt.Printf("%02x  ", m.Data[i+j])
+		}
+		for j := rowcount; j < 8; j++ {
+			fmt.Printf("    ")
+		}
+		fmt.Printf("  '%s'\n", viewString(m.Data[i:(i+rowcount)]))
+	}
+}
+
+//endTools
+
 // TransactionStatus is the current transaction state.
 type TransactionStatus byte
 
@@ -98,6 +172,12 @@ func (e *Encoder) i32(i int32) error {
 	return err
 }
 
+func (e *Encoder) u32(i uint32) error {
+	binary.BigEndian.PutUint32(e.scratch[:], i)
+	_, err := e.buf.Write(e.scratch[:])
+	return err
+}
+
 // ReadyForQuery encodes a "ready for query" message.
 func (e *Encoder) ReadyForQuery(status TransactionStatus) (Message, error) {
 	e.buf.Reset()
@@ -149,6 +229,9 @@ const (
 
 	// NoticeFieldHint is a suggestion of how to address the issue.
 	NoticeFieldHint NoticeFieldType = 'H'
+
+	// NoticeFieldHint the SQLSTATE code for the error (see Appendix A). Not localizable. Always present.
+	NoticeFieldCode NoticeFieldType = 'C'
 )
 
 // NoticeField is a field in an error or notice.
@@ -365,3 +448,95 @@ func (e *Encoder) BackendKeyData(pid, key int32) (Message, error) {
 		Data: e.buf.Bytes(),
 	}, nil
 }
+
+func (e *Encoder) ParameterStatus(param, value string) (Message, error) {
+	e.buf.Reset()
+	//param + NULL + value+ NULL
+	_, err := e.buf.WriteString(param)
+	if err != nil {
+		return Message{}, err
+	}
+	err = e.buf.WriteByte(0)
+	if err != nil {
+		return Message{}, err
+	}
+	_, err = e.buf.WriteString(value)
+	if err != nil {
+		return Message{}, err
+	}
+	err = e.buf.WriteByte(0)
+	if err != nil {
+		return Message{}, err
+	}
+	return Message{
+		Type: TypeParameterStatus,
+		Data: e.buf.Bytes(),
+	}, nil
+}
+
+type SimpleColumn struct {
+	Name    string
+	Typeid  int32
+	Typelen int16
+}
+
+func (e *Encoder) EncodeColumn(name string, typeid int32, typelen int16) (Message, error) {
+	return e.EncodeColumns(SimpleColumn{
+		Name:    name,
+		Typeid:  typeid,
+		Typelen: typelen,
+	})
+}
+func (e *Encoder) EncodeColumns(cols ...SimpleColumn) (Message, error) {
+
+	e.buf.Reset()
+	err := e.i16(int16(len(cols))) // number of columns in result
+	if err != nil {
+		return Message{}, nil
+	}
+	for _, col := range cols {
+
+		_, err = e.buf.WriteString(col.Name) // column name
+		if err != nil {
+			return Message{}, err
+		}
+		err = e.buf.WriteByte(0) //null terminate
+		if err != nil {
+			return Message{}, err
+		}
+
+		err = e.i32(0) //tabel id
+		if err != nil {
+			return Message{}, err
+		}
+
+		err = e.i16(0) //field id(attnum)
+		if err != nil {
+			return Message{}, err
+		}
+
+		err = e.i32(col.Typeid) //type_id
+		if err != nil {
+			return Message{}, err
+		}
+
+		err = e.i16(col.Typelen) //type_len
+		if err != nil {
+			return Message{}, err
+		}
+
+		err = e.i32(-1) //type_mod
+		if err != nil {
+			return Message{}, err
+		}
+
+		err = e.i16(0) //format  0 text 1 binary
+		if err != nil {
+			return Message{}, err
+		}
+	}
+	return Message{
+		Type: TypeRowDescription,
+		Data: e.buf.Bytes(),
+	}, nil
+}
diff --git a/pg/protocol.go b/pg/protocol.go
index 222a9055f..754e128e3 100644
--- a/pg/protocol.go
+++ b/pg/protocol.go
@@ -30,7 +30,10 @@ import (
 	"time"
 
 	"github.com/molecula/featurebase/v2/pg/message"
+	"github.com/molecula/featurebase/v2/sql"
+	"github.com/molecula/featurebase/v2/vprint"
 	"github.com/pkg/errors"
+	"vitess.io/vitess/go/vt/sqlparser"
 )
 
 // Protocol is a Postgres protocol version.
@@ -48,6 +51,9 @@ const (
 
 	// ProtocolSupported is the main protocol version supported by this package.
 	ProtocolSupported Protocol = ProtocolPostgres30
+
+	// PgServerVersion is the latest version of postgres that we claim to support.
+	PgServerVersion = "13.0.0"
 )
 
 // Major returns the major revision of the protocol.
@@ -170,6 +176,7 @@ startup:
 		// Reject the unsecured connection.
 		return errors.Errorf("client at %s attempted to initiate an unsecured postgres conenction", conn.RemoteAddr())
 	}
+	vprint.VV("TProcotcol: %v", proto)
 
 	switch proto {
 	case ProtocolCancel:
@@ -209,6 +216,7 @@ func parseParams(data []byte) (map[string]string, error) {
 
 // handleCancel handles cancel request connections.
 func (s *Server) handleCancel(ctx context.Context, conn net.Conn, data []byte) error {
+	vprint.VV("handle cancel")
 	if len(data) != 8 {
 		return errors.New("malformed cancellation packet")
 	}
@@ -220,6 +228,7 @@ func (s *Server) handleCancel(ctx context.Context, conn net.Conn, data []byte) e
 	pid := int32(binary.BigEndian.Uint32(data[:4]))
 	key := int32(binary.BigEndian.Uint32(data[4:]))
 
+	vprint.VV("cancel pid:%v key:%v", pid, key)
 	err := s.CancellationManager.Cancel(CancellationToken{PID: pid, Key: key})
 	switch err {
 	case nil:
@@ -233,6 +242,370 @@ func (s *Server) handleCancel(ctx context.Context, conn net.Conn, data []byte) e
 
 	return nil
 }
+func (s *Server) SendParameterStatus(w *message.WireWriter, param, value string, encoder *message.Encoder) error {
+	msg, err := encoder.ParameterStatus(param, value)
+	if err != nil {
+		return err
+	}
+	err = w.WriteMessage(msg)
+	if err != nil {
+		return errors.Wrap(err, "sending parameter status")
+	}
+	return nil
+}
+
+type Result struct {
+}
+type PgType byte
+
+const (
+	pgNotPg          PgType = 'x'
+	pgBackendPid     PgType = 'a'
+	pgVersion        PgType = 'b'
+	pgCountType      PgType = 'c'
+	pgQueryTime      PgType = 'd'
+	pgTerminate      PgType = 'e'
+	pgEmpty          PgType = 'f'
+	pgSetApplication PgType = 'g'
+	pgSelect1        PgType = 'h'
+	pgSchema         PgType = 'i'
+	pgBegin          PgType = 'j'
+)
+
+type Portal struct {
+	Name         string
+	Writer       *message.WireWriter
+	commands     []message.Message
+	Encoder      *message.Encoder
+	mapper       *sql.Mapper
+	results      []Result
+	sql          string
+	pgspecial    PgType
+	pid          int32
+	queryStart   time.Time
+	server       *Server
+	cancelNotify <-chan struct{}
+}
+
+func (p *Portal) Reset() {
+	vprint.VV("Portal Reset")
+	p.Name = ""
+	p.sql = ""
+	p.pgspecial = pgNotPg
+	p.commands = p.commands[:0]
+}
+
+func (p *Portal) Bind() {
+	p.Add(message.BindComplete)
+}
+func (p *Portal) Parse(data []byte) {
+	p.queryStart = time.Now()
+	b := bytes.Trim(data, "\x00")
+	vprint.VV("PARSE RAW: (%v) (%d)", string(b), len(b))
+	if strings.Contains(string(b), "EXTRACT") {
+		// had to add this hack because the vitis parser doesn't handle...
+		/*
+		  	SELECT pid as id,
+		   	 query as stmt,
+		     EXTRACT(seconds from query_start - NOW()) as elapsed_time
+		     FROM pg_stat_activity
+		     WHERE usename='docker'`
+		*/
+		p.pgspecial = pgQueryTime
+		p.Name = "SELECT"
+		p.sql = string(b)
+		p.Add(message.ParseOK)
+		return
+	}
+
+	if len(b) > 2 {
+
+		query, err := p.mapper.MapSQL(string(b))
+		if err != nil {
+			vprint.VV("Parse Err: '%v'", err)
+		} else {
+
+			vprint.VV("TODD Query: %#v", query.SQL)
+			if strings.Contains(strings.ToLower(query.SQL), "select 1") {
+				vprint.VV("SQL1")
+				p.pgspecial = pgSelect1
+				p.Name = "SELECT"
+			} else {
+				switch query.SQLType {
+				case sql.SQLTypeSet:
+					p.Name = "SET"
+					set := query.Statement.(*sqlparser.Set)
+					for _, item := range set.Exprs {
+						vprint.VV("SET name=(%v) ", item.Name)
+						if item.Name.String() == "application_name" {
+							vprint.VV("SET expr=(%#v) ", item.Expr)
+							switch val := item.Expr.(type) {
+							case *sqlparser.SQLVal:
+								vprint.VV("getting %v (%v)", val.Type, string(val.Val))
+								p.pgspecial = pgSetApplication
+							}
+						}
+					}
+				case sql.SQLTypeSelect:
+					p.Name = "SELECT"
+					p.pgspecial = pgNotPg
+					stmt := query.Statement.(*sqlparser.Select)
+					for _, item := range stmt.SelectExprs {
+						switch expr := item.(type) {
+						case *sqlparser.AliasedExpr:
+							switch colExpr := expr.Expr.(type) {
+							case *sqlparser.FuncExpr:
+								funcName := strings.ToLower(colExpr.Name.String())
+								switch funcName {
+								case "pg_backend_pid":
+									//SELECT pg_backend_pid()
+									p.pgspecial = pgBackendPid
+									//p.encoder.RowDescription() //TODO(twg) move to the right spt
+								case "pg_terminate_backend":
+									//need the arg 100
+									//select pg_terminate_backend(100)
+									vprint.VV("terminate %#v", colExpr.Exprs)
+									//colExpr.Exprs[0]
+									p.pgspecial = pgTerminate
+								case "version":
+									//SELECT VERSION() AS version
+									p.pgspecial = pgVersion
+								}
+								//need to return  the pid from the cancelation object
+								//add row description object
+								//add data row for item
+							}
+						}
+
+					}
+					for _, item := range stmt.From {
+						switch from := item.(type) {
+						case *sqlparser.AliasedTableExpr:
+							tableName := from.Expr.(sqlparser.TableName).ToViewName().Name.String()
+							vprint.VV("looking at (%v)", tableName)
+							switch tableName {
+							case "pg_type":
+								p.pgspecial = pgCountType
+							case "pg_stat_activity":
+								p.pgspecial = pgQueryTime
+							case "tables":
+								p.pgspecial = pgSchema
+							}
+						}
+					}
+					p.sql = string(b)
+				case sql.SQLTypeBegin:
+					p.pgspecial = pgBegin
+					//				panic("TODO")
+					// ingnore for now
+				case sql.SQLTypeShow:
+					p.Name = "SHOW"
+					p.pgspecial = pgNotPg
+					p.sql = string(b)
+				}
+			}
+		}
+		// TODO (twg) hack parsing for short term
+		//	parts := strings.Split(strings.TrimSuffix(string(data), "\x00"), " ")
+		//	p.Name = parts[0][1:] //TODO (twg) major hack
+		vprint.VV("PG SET:(%v)", p.pgspecial)
+	} else {
+		vprint.VV("SETTING EMPTY")
+		p.pgspecial = pgEmpty
+	}
+	p.Add(message.ParseOK)
+}
+func (p *Portal) Describe() {
+	/*
+		if p.pgspecial != pgNotPg {
+			//do custom handling for setup
+		}
+		if len(p.results) == 0 {
+			p.Add(&message.NoData)
+			p.Add(&message.EmptyQueryResponse)
+			return
+		}
+	*/
+
+}
+func (p *Portal) Execute() (shouldTerminate bool, queryReady bool) {
+	queryReady = true
+	vprint.VV("port.Execute: (%v) q=(%v)", p.pgspecial, p.sql)
+	switch p.pgspecial {
+	case pgBackendPid:
+		rowDescription, err := p.Encoder.EncodeColumn("pg_backend_pid", int32(23), 4)
+		if err != nil {
+			panic(err)
+		}
+		p.Add(rowDescription)
+		pid := fmt.Sprintf("%v", p.pid)
+		dataRow, _ := p.Encoder.TextRow(pid)
+		p.Add(dataRow)
+		//needs data row with cancel token
+	case pgVersion:
+		rowDescription, err := p.Encoder.EncodeColumn("version", int32(25), -1)
+		if err != nil {
+			panic(err)
+		}
+		p.Add(rowDescription)
+		dataRow, _ := p.Encoder.TextRow("PostgresSQL 13.0 (molecula)")
+		p.Add(dataRow)
+	case pgSelect1:
+		rowDescription, err := p.Encoder.EncodeColumn("?column?", int32(23), 4)
+		if err != nil {
+			panic(err)
+		}
+		p.Add(rowDescription)
+		dataRow, _ := p.Encoder.TextRow("1")
+		p.Add(dataRow)
+	case pgCountType:
+		//need to block
+		vprint.VV("blocking til terminated")
+		<-p.server.lookerChannel
+		vprint.VV("Released")
+		//now need to send ParseComplete,BindComplete,RowDescription,Error(Terminate)
+		//TODO(twg) handle the error
+		rowDescription, err := p.Encoder.EncodeColumn("count", int32(20), 8)
+		if err != nil {
+			panic(err)
+		}
+		p.Add(rowDescription)
+		errorResponse, _ := p.Encoder.Error(
+			message.NoticeField{
+				Type: message.NoticeFieldSeverity,
+				Data: "FATAL",
+			},
+			message.NoticeField{
+				Type: message.NoticeFieldMessage,
+				Data: "terminating connection due to administrator command",
+			},
+			message.NoticeField{
+				Type: message.NoticeFieldCode,
+				Data: "57P01",
+			},
+		)
+		p.Add(errorResponse)
+		p.Sync() //send and Reset
+		return true, queryReady
+	case pgQueryTime:
+		// need to return something so that the id can be queried
+		//need to return  SELECT pid as id, query as stmt, EXTRACT(seconds from query_start - NOW()) as elapsed_time          FROM pg_stat_activity
+		//seems like we need a map of pids to querys
+		p.server.dumpPortalsTo(p)
+	case pgTerminate:
+		//note just have 1 lock that blocks all who try to count the activities
+		vprint.VV("removing the block ")
+		//TODO (twg) lock this
+		close(p.server.lookerChannel)                //release all the other blockers and allow them to terminate
+		p.server.lookerChannel = make(chan struct{}) //create a new one just in case
+		// i think it needs to return boolean true
+		rowDescription, err := p.Encoder.EncodeColumn("pg_terminate_backend", int32(16), 1)
+		if err != nil {
+			panic(err)
+		}
+		p.Add(rowDescription)
+		dataRow, _ := p.Encoder.TextRow("t")
+		p.Add(dataRow)
+		commandComplete, _ := p.Encoder.CommandComplete("SELECT 1")
+		p.Add(commandComplete)
+		p.Sync()
+		return false, queryReady
+	case pgEmpty:
+		p.Add(message.NoData)
+		p.Add(message.EmptyQueryResponse)
+		p.Sync()
+		queryReady = true
+		return false, queryReady
+	case pgSetApplication:
+		//needs to add/send status
+		msg, err := p.Encoder.ParameterStatus("application_name", "PostgreSQL JDBC Driver") //TODO (twg) should be saved from parse
+		if err != nil {
+			return
+		}
+		p.Add(msg)
+	case pgSchema:
+		//need to add the descrition for the 3 fields
+		//	---[Field 01]---  name='table_schema'  type=19  type_len=64  type_mod=4294967295  relid=13276  attnum=2  format=0
+		// ---[Field 02]---  name='table_name'  type=19  type_len=64  type_mod=4294967295  relid=13276  attnum=3  format=0
+		parts := []message.SimpleColumn{
+			{
+				Name:    "table_schema",
+				Typeid:  int32(19),
+				Typelen: 64,
+			},
+			{
+				Name:    "table_name",
+				Typeid:  int32(19),
+				Typelen: 64,
+			},
+		}
+		rowDescription, err := p.Encoder.EncodeColumns(parts...)
+		if err != nil { //TODO (twg) shadowing err
+			return
+		}
+		p.Add(rowDescription)
+		err = p.HandleSchema()
+		if err != nil {
+			return
+		}
+		vprint.VV("SHOULd make schema")
+
+	case pgNotPg:
+		vprint.VV("GOOOO>(%v)", p.sql)
+		query := SimpleQuery(p.sql)
+		err := p.server.handleQuery(p, query, p.cancelNotify)
+		if err != nil {
+			return
+		}
+
+	case pgBegin:
+		vprint.VV("BEGIN")
+		p.Name = "BEGIN"
+	}
+
+	//maybe add in the number of items in select clause
+	if len(p.Name) > 0 { //only send command complete for those that have names
+		vprint.VV("EXECING NAME: '%v'", p.Name)
+		message, _ := p.Encoder.CommandComplete(p.Name)
+		vprint.VV("AFTER")
+		p.Add(message)
+	}
+	return
+}
+
+// handleStandard handles a connection in the standard postgres wire protocol.
+func (p *Portal) Sync() {
+	for _, m := range p.commands {
+		vprint.VV("Sending: '%v'", m.Type)
+		p.Writer.WriteMessage(m)
+	}
+	p.Writer.Flush()
+	p.Reset()
+}
+func (p *Portal) Add(m message.Message) {
+	cp := message.Message{Type: m.Type, Data: make([]byte, len(m.Data))}
+	copy(cp.Data, m.Data)
+	p.commands = append(p.commands, cp)
+}
+
+func (p *Portal) DumpComands() {
+
+	for _, m := range p.commands {
+		m.Dump("DUMPING:")
+	}
+}
+func (p *Portal) HandleSchema() error {
+	return p.server.QueryHandler.HandleSchema(context.Background(), p)
+}
+
+//implement the QueryResultWriterqueryResultR
+func (p *Portal) WriteMessage(m message.Message) error {
+	p.Add(m)
+	return nil
+}
+func (p *Portal) Flush() error {
+	return nil
+}
 
 // handleStandard handles a connection in the standard postgres wire protocol.
 // The client is responsible for closing the connection when this finishes.
@@ -255,7 +628,7 @@ func (s *Server) handleStandard(ctx context.Context, proto Protocol, conn net.Co
 	if err != nil {
 		return errors.Wrap(err, "parsing parameters")
 	}
-
+	fmt.Printf("postgres connection params\n%#v\n", params) //TODO (twg) remove
 	if user, ok := params["user"]; ok {
 		// Log the connection.
 		s.Logger.Debugf("new postgres connection from user %q at %v", user, conn.RemoteAddr())
@@ -322,8 +695,57 @@ func (s *Server) handleStandard(ctx context.Context, proto Protocol, conn net.Co
 	if err != nil {
 		return errors.Wrap(err, "sending authentication confirmation")
 	}
+	err = s.SendParameterStatus(w, "application_name", "", &encoder)
+	if err != nil {
+		return errors.Wrap(err, "sending parameter status server version")
+	}
+	err = s.SendParameterStatus(w, "client_encoding", "UTF8", &encoder)
+	if err != nil {
+		return errors.Wrap(err, "sending parameter status server version")
+	}
+	err = s.SendParameterStatus(w, "DateStyle", "ISO, MDY", &encoder)
+	if err != nil {
+		return errors.Wrap(err, "sending parameter status server version")
+	}
+	err = s.SendParameterStatus(w, "integer_datetimes", "on", &encoder)
+	if err != nil {
+		return errors.Wrap(err, "sending parameter status server version")
+	}
+	err = s.SendParameterStatus(w, "IntervalStyle", "postgres", &encoder)
+	if err != nil {
+		return errors.Wrap(err, "sending parameter status server version")
+	}
+	err = s.SendParameterStatus(w, "is_superuser", "on", &encoder)
+	if err != nil {
+		return errors.Wrap(err, "sending parameter status server version")
+	}
+	err = s.SendParameterStatus(w, "server_encoding", "UTF8", &encoder)
+	if err != nil {
+		return errors.Wrap(err, "sending parameter status server version")
+	}
+	err = s.SendParameterStatus(w, "server_version", PgServerVersion, &encoder)
+	if err != nil {
+		return errors.Wrap(err, "sending parameter status server version")
+	}
+	/*
+		-PARAMETER STATUS name='TimeZone', value='GMT'
+	*/
+	err = s.SendParameterStatus(w, "session_authorization", "docker", &encoder) //TODO(twg) figure out valid values here
+	if err != nil {
+		return errors.Wrap(err, "sending parameter status server version")
+	}
+	err = s.SendParameterStatus(w, "standard_conforming_strings", "on", &encoder) //TODO(twg) figure out valid values here
+	if err != nil {
+		return errors.Wrap(err, "sending parameter status server version")
+	}
+	err = s.SendParameterStatus(w, "TimeZone", "GMT", &encoder) //TODO(twg) figure out valid values here
+	if err != nil {
+		return errors.Wrap(err, "sending parameter status server version")
+	}
 
+	vprint.VV("CancellationManger: %v", s.CancellationManager)
 	var cancelNotify <-chan struct{}
+	var pid int32
 	if s.CancellationManager != nil {
 		notify, cancel, token, err := s.CancellationManager.Token()
 		if err != nil {
@@ -332,6 +754,7 @@ func (s *Server) handleStandard(ctx context.Context, proto Protocol, conn net.Co
 		defer cancel()
 
 		msg, err := encoder.BackendKeyData(token.PID, token.Key)
+		pid = token.PID
 		if err != nil {
 			return errors.Wrap(err, "encoding cancellation key data")
 		}
@@ -343,11 +766,25 @@ func (s *Server) handleStandard(ctx context.Context, proto Protocol, conn net.Co
 	}
 
 	var queryReady bool
+	portal := &Portal{
+		Writer:       w,
+		Encoder:      &encoder,
+		commands:     make([]message.Message, 0),
+		mapper:       sql.NewMapper(),
+		pid:          pid,
+		server:       s,
+		cancelNotify: cancelNotify,
+	}
+	s.addPortal(portal)
+	defer s.removePortal(portal)
+	//mapper.Logger = logger
 	for {
 		if !queryReady {
 			// Indicate that we are ready for a query.
 			// TODO: provide a valid transaction state.
-			msg, err := encoder.ReadyForQuery(message.TransactionStatusActive)
+			//msg, err := encoder.ReadyForQuery(message.TransactionStatusActive)
+			portal.sql = ""
+			msg, err := encoder.ReadyForQuery(message.TransactionStatusIdle)
 			if err != nil {
 				return errors.Wrap(err, "sending query ready status")
 			}
@@ -373,10 +810,12 @@ func (s *Server) handleStandard(ctx context.Context, proto Protocol, conn net.Co
 				// However, it seems that no clients completely follow the spec, so we shouldn't rely on anything that isn't entirely straightforward.
 				s.Logger.Debugf("postgres client sent additional data without waiting for completion")
 			}
+			queryReady = true
 		}
 
 		// Read the next packet.
 		msg, err := r.ReadMessage()
+		msg.Dump("start-") // TODO(twg) remove
 		if err != nil {
 			if err == errPreempted {
 				// The server is shutting down.
@@ -406,6 +845,26 @@ func (s *Server) handleStandard(ctx context.Context, proto Protocol, conn net.Co
 			// We are done.
 			return w.Flush()
 
+		case message.TypeParse:
+			fmt.Println("Type Parse", strings.Trim(string(msg.Data), "\x00"))
+			portal.Parse(msg.Data)
+		case message.TypeBind:
+			fmt.Println("Type Bind", strings.Trim(string(msg.Data), "\x00"))
+			//TODO(twg) apply values to bindings in the future
+			// Parse the query message (a null-terminated string).
+			portal.Bind()
+		case message.TypeExecute:
+			fmt.Println("Type Execute", strings.Trim(string(msg.Data), "\x00"))
+			//term, qr := portal.Execute()
+			term, qr := portal.Execute()
+			if term {
+				return w.Flush()
+			}
+			queryReady = qr
+		case message.TypeSync:
+			fmt.Println("Type Sync", strings.Trim(string(msg.Data), "\x00"))
+			portal.Sync()
+			queryReady = false
 		case message.TypeSimpleQuery:
 			// Execute a simple query.
 
@@ -419,10 +878,23 @@ func (s *Server) handleStandard(ctx context.Context, proto Protocol, conn net.Co
 			if err != nil {
 				return err
 			}
-
+			label := "N/A"
+			switch msg.Data[0] {
+			case 'S': //prepared statement
+				label = "prepared statement"
+			case 'P': //portal
+				label = "portal"
+			}
+			vprint.VV("describe %v:'%v'", label, msg.Data[1:])
+		case message.TypeDescribe:
+			//TODO (twg) major hack alet
+			portal.Describe()
+		case message.TypeClose:
+			return w.Flush()
 		default:
 			// The message is not supported yet.
 			// Send an error.
+			vprint.VV("unrecognized postgres packet %v", msg)
 			s.Logger.Errorf("unrecognized postgres packet %v", msg)
 			msg, err = encoder.Error(
 				message.NoticeField{
@@ -452,6 +924,71 @@ func (s *Server) handleStandard(ctx context.Context, proto Protocol, conn net.Co
 		}
 	}
 }
+func (s *Server) addPortal(p *Portal) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.portals = append(s.portals, p)
+}
+func (s *Server) removePortal(p *Portal) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	for i, portal := range s.portals {
+		if portal.pid == p.pid {
+			//remove i
+			s.portals = append(s.portals[:i], s.portals[i+1:]...)
+			return
+		}
+
+	}
+
+}
+func (s *Server) dumpPortalsTo(p *Portal) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	//need to add the descrition for the 3 fields
+	//	<-:-ROW DESCRIPTION: num_fields=3
+	//---[Field 01]---  name='id'  type=23  type_len=4  type_mod=4294967295  relid=12250  attnum=3  format=0
+	//---[Field 02]---  name='stmt'  type=25  type_len=65535  type_mod=4294967295  relid=12250  attnum=20  format=0  -
+	//--[Field 03]---  name='elapsed_time'  type=701  type_len=8  type_mod=4294967295  relid=0  attnum=0  format=0
+	parts := []message.SimpleColumn{
+		{
+			Name:    "id",
+			Typeid:  int32(23),
+			Typelen: 4,
+		},
+		{
+			Name:    "stmt",
+			Typeid:  int32(25),
+			Typelen: -1,
+		},
+		{
+			Name:    "elapsed_time",
+			Typeid:  int32(701),
+			Typelen: 8,
+		},
+	}
+	rowDescription, err := p.Encoder.EncodeColumns(parts...)
+	if err != nil {
+		return err
+	}
+	p.Add(rowDescription)
+
+	for _, portal := range s.portals {
+		vprint.VV("port: %v (%v) %v", portal.pid, portal.sql, time.Since(portal.queryStart))
+		dataRow, err := p.Encoder.TextRow(
+			fmt.Sprintf("%v", portal.pid),
+			portal.sql,
+			fmt.Sprintf("%v", time.Since(portal.queryStart).Seconds()))
+		if err != nil {
+			return err
+		}
+		//if sql == "" need to put in a null record
+		//need to add the dararow
+		//also need to figure out null types
+		p.Add(dataRow)
+	}
+	return nil
+}
 
 // handleQuery processes a single query on a connection.
 func (s *Server) handleQuery(w message.Writer, query Query, cancelNotify <-chan struct{}) error {
diff --git a/pg/query.go b/pg/query.go
index cc81d490f..7aa231670 100644
--- a/pg/query.go
+++ b/pg/query.go
@@ -60,6 +60,7 @@ type QueryResultWriter interface {
 type QueryHandler interface {
 	// HandleQuery executes a query and writes the results back.
 	HandleQuery(context.Context, QueryResultWriter, Query) error
+	HandleSchema(context.Context, *Portal) error
 }
 
 // queryResultWriter implements QueryResultWrtiter over postgres wire protocol.
diff --git a/pg/server.go b/pg/server.go
index 071024385..28aa6ca24 100644
--- a/pg/server.go
+++ b/pg/server.go
@@ -59,6 +59,9 @@ type Server struct {
 	// CancellationManager is the cancellation manager to use.
 	// If this is not set, no cancellations will be applied.
 	CancellationManager CancellationManager
+	lookerChannel       chan struct{}
+	mu                  sync.Mutex
+	portals             []*Portal
 }
 
 // ServeConn serves a single connection.
@@ -79,6 +82,8 @@ func (s *Server) Serve(ctx context.Context, l net.Listener) (err error) {
 			err = cerr
 		}
 	}(ctx)
+	// TODO (twg) added for looker
+	s.lookerChannel = make(chan struct{})
 
 	// Wait for the listener to be closed and all connections to shut down.
 	var wg sync.WaitGroup
diff --git a/server/pg.go b/server/pg.go
index 0f03e7dfb..dc61e6262 100644
--- a/server/pg.go
+++ b/server/pg.go
@@ -25,7 +25,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/molecula/featurebase/v2"
+	pilosa "github.com/molecula/featurebase/v2"
 	"github.com/molecula/featurebase/v2/logger"
 	"github.com/molecula/featurebase/v2/pg"
 	"github.com/molecula/featurebase/v2/pql"
@@ -67,9 +67,9 @@ func NewPostgresServer(api *pilosa.API, logger logger.Logger, tls *tls.Config) *
 
 // NewPostgresHandler creates a postgres query handler wrapping the pilosa API.
 func NewPostgresHandler(api *pilosa.API, logger logger.Logger) pg.QueryHandler {
-	return &queryDecodeHandler{
-		child: &pilosaQueryHandler{
-			api:    api,
+	return &QueryDecodeHandler{
+		Child: &PilosaQueryHandler{
+			Api:    api,
 			logger: logger,
 		},
 	}
@@ -107,6 +107,10 @@ func (s *PostgresServer) Close() error {
 	return s.eg.Wait()
 }
 
+func (s *PostgresServer) GetAPI() *pilosa.API {
+	return s.api
+}
+
 type pgPQLQuery struct {
 	index string
 	query string
@@ -135,8 +139,8 @@ func pgDecodePQL(str string) (q pg.Query, err error) {
 	}, nil
 }
 
-type pilosaQueryHandler struct {
-	api    *pilosa.API
+type PilosaQueryHandler struct {
+	Api    *pilosa.API
 	logger logger.Logger
 }
 
@@ -449,10 +453,10 @@ func pgWriteResult(w pg.QueryResultWriter, result interface{}) error {
 	}
 }
 
-func (pqh *pilosaQueryHandler) HandleQuery(ctx context.Context, w pg.QueryResultWriter, q pg.Query) error {
+func (pqh *PilosaQueryHandler) HandleQuery(ctx context.Context, w pg.QueryResultWriter, q pg.Query) error {
 	switch q := q.(type) {
 	case pgPQLQuery:
-		resp, err := pqh.api.Query(ctx, &pilosa.QueryRequest{
+		resp, err := pqh.Api.Query(ctx, &pilosa.QueryRequest{
 			Index: q.index,
 			Query: q.query,
 		})
@@ -465,7 +469,7 @@ func (pqh *pilosaQueryHandler) HandleQuery(ctx context.Context, w pg.QueryResult
 		return errors.Wrap(pgWriteResult(w, resp.Results[0]), "writing query result")
 
 	case pg.SimpleQuery:
-		resp, err := execSQL(ctx, pqh.api, pqh.logger, string(q))
+		resp, err := execSQL(ctx, pqh.Api, pqh.logger, string(q))
 		if err != nil {
 			return errors.Wrap(err, "executing query")
 		}
@@ -476,11 +480,11 @@ func (pqh *pilosaQueryHandler) HandleQuery(ctx context.Context, w pg.QueryResult
 	}
 }
 
-type queryDecodeHandler struct {
-	child pg.QueryHandler
+type QueryDecodeHandler struct {
+	Child pg.QueryHandler
 }
 
-func (qdh *queryDecodeHandler) HandleQuery(ctx context.Context, w pg.QueryResultWriter, q pg.Query) error {
+func (qdh *QueryDecodeHandler) HandleQuery(ctx context.Context, w pg.QueryResultWriter, q pg.Query) error {
 	switch qv := q.(type) {
 	case pg.SimpleQuery:
 		if strings.HasPrefix(string(qv), "[") {
@@ -492,5 +496,24 @@ func (qdh *queryDecodeHandler) HandleQuery(ctx context.Context, w pg.QueryResult
 		}
 	}
 
-	return qdh.child.HandleQuery(ctx, w, q)
+	return qdh.Child.HandleQuery(ctx, w, q)
+}
+
+func (pqh *PilosaQueryHandler) HandleSchema(ctx context.Context, portal *pg.Portal) error {
+	schema, err := pqh.Api.Schema(context.Background(), false)
+	if err != nil {
+		return err
+	}
+	for _, ii := range schema {
+		dataRow, err := portal.Encoder.TextRow("featurebase", ii.Name)
+		if err != nil {
+			return err
+		}
+		portal.Add(dataRow)
+	}
+	return nil
+}
+
+func (qdh *QueryDecodeHandler) HandleSchema(ctx context.Context, portal *pg.Portal) error {
+	return qdh.Child.HandleSchema(ctx, portal)
 }
diff --git a/sql/mapper.go b/sql/mapper.go
index 3ff500d8f..80ce64b6c 100644
--- a/sql/mapper.go
+++ b/sql/mapper.go
@@ -25,6 +25,8 @@ import (
 const (
 	SQLTypeSelect = "select"
 	SQLTypeShow   = "show"
+	SQLTypeSet    = "set"
+	SQLTypeBegin  = "begin"
 	SQLTypeEmpty  = ""
 )
 
@@ -101,6 +103,10 @@ func (m *Mapper) MapSQL(sql string) (*MappedSQL, error) {
 		}
 	case *sqlparser.Show:
 		sqlType = SQLTypeShow
+	case *sqlparser.Set:
+		sqlType = SQLTypeSet
+	case *sqlparser.Begin:
+		sqlType = SQLTypeBegin
 	}
 
 	return &MappedSQL{

From 419d1ed05dbf7ac802a36ff1cc677b04e19b13fb Mon Sep 17 00:00:00 2001
From: Todd Gruben <todd@pilosa.com>
Date: Mon, 13 Sep 2021 14:16:20 -0500
Subject: [PATCH 25/66] linter cleanup

---
 pg/message/message.go |  2 ++
 pg/pgtest/handler.go  |  3 +++
 pg/protocol.go        | 25 +++++++++++++++++--------
 3 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/pg/message/message.go b/pg/message/message.go
index 5d6f67014..757053297 100644
--- a/pg/message/message.go
+++ b/pg/message/message.go
@@ -172,11 +172,13 @@ func (e *Encoder) i32(i int32) error {
 	return err
 }
 
+/* removed for linter now
 func (e *Encoder) u32(i uint32) error {
 	binary.BigEndian.PutUint32(e.scratch[:], i)
 	_, err := e.buf.Write(e.scratch[:])
 	return err
 }
+*/
 
 // ReadyForQuery encodes a "ready for query" message.
 func (e *Encoder) ReadyForQuery(status TransactionStatus) (Message, error) {
diff --git a/pg/pgtest/handler.go b/pg/pgtest/handler.go
index b146359d8..02b1361e2 100644
--- a/pg/pgtest/handler.go
+++ b/pg/pgtest/handler.go
@@ -30,6 +30,9 @@ type HandlerFunc func(context.Context, pg.QueryResultWriter, pg.Query) error
 func (h HandlerFunc) HandleQuery(ctx context.Context, w pg.QueryResultWriter, q pg.Query) error {
 	return h(ctx, w, q)
 }
+func (h HandlerFunc) HandleSchema(ctx context.Context, portal *pg.Portal) error {
+	return nil
+}
 
 var _ pg.QueryHandler = HandlerFunc(nil)
 
diff --git a/pg/protocol.go b/pg/protocol.go
index 754e128e3..377e2ac16 100644
--- a/pg/protocol.go
+++ b/pg/protocol.go
@@ -273,12 +273,12 @@ const (
 )
 
 type Portal struct {
-	Name         string
-	Writer       *message.WireWriter
-	commands     []message.Message
-	Encoder      *message.Encoder
-	mapper       *sql.Mapper
-	results      []Result
+	Name     string
+	Writer   *message.WireWriter
+	commands []message.Message
+	Encoder  *message.Encoder
+	mapper   *sql.Mapper
+	//results      []Result
 	sql          string
 	pgspecial    PgType
 	pid          int32
@@ -491,7 +491,11 @@ func (p *Portal) Execute() (shouldTerminate bool, queryReady bool) {
 		// need to return something so that the id can be queried
 		//need to return  SELECT pid as id, query as stmt, EXTRACT(seconds from query_start - NOW()) as elapsed_time          FROM pg_stat_activity
 		//seems like we need a map of pids to querys
-		p.server.dumpPortalsTo(p)
+		err := p.server.dumpPortalsTo(p)
+		if err != nil {
+			return
+		}
+
 	case pgTerminate:
 		//note just have 1 lock that blocks all who try to count the activities
 		vprint.VV("removing the block ")
@@ -577,7 +581,12 @@ func (p *Portal) Execute() (shouldTerminate bool, queryReady bool) {
 func (p *Portal) Sync() {
 	for _, m := range p.commands {
 		vprint.VV("Sending: '%v'", m.Type)
-		p.Writer.WriteMessage(m)
+		err := p.Writer.WriteMessage(m)
+		if err != nil {
+			//TODO (twg) need to change signature to return error
+			vprint.VV("error %v", err)
+			return
+		}
 	}
 	p.Writer.Flush()
 	p.Reset()

From c56fd339247c2d8f43b7b213a8029951d6e72052 Mon Sep 17 00:00:00 2001
From: Todd Gruben <todd@pilosa.com>
Date: Fri, 17 Sep 2021 09:52:57 -0500
Subject: [PATCH 26/66] wip

---
 api.go       |  5 +++
 server/pg.go | 90 +++++++++++++++++++++++++++++++++++++++++++++++++---
 2 files changed, 91 insertions(+), 4 deletions(-)

diff --git a/api.go b/api.go
index 8e7fa7b2b..ce8ffda33 100644
--- a/api.go
+++ b/api.go
@@ -36,6 +36,8 @@ import (
 
 	"github.com/molecula/featurebase/v2/disco"
 	"github.com/molecula/featurebase/v2/ingest"
+
+	//"github.com/molecula/featurebase/v2/pg"
 	"github.com/molecula/featurebase/v2/pql"
 	"github.com/molecula/featurebase/v2/roaring"
 	"github.com/molecula/featurebase/v2/stats"
@@ -3024,6 +3026,9 @@ processing:
 	}
 	return result, ctx.Err()
 }
+func (api *API) Plan(ctx context.Context, q string) (*Stmt, error) {
+	return api.server.PlanSQL(ctx, q)
+}
 
 type serverInfo struct {
 	ShardWidth       uint64 `json:"shardWidth"`
diff --git a/server/pg.go b/server/pg.go
index dc61e6262..a6395adff 100644
--- a/server/pg.go
+++ b/server/pg.go
@@ -28,6 +28,8 @@ import (
 	pilosa "github.com/molecula/featurebase/v2"
 	"github.com/molecula/featurebase/v2/logger"
 	"github.com/molecula/featurebase/v2/pg"
+
+	//"github.com/molecula/featurebase/v2/pg"
 	"github.com/molecula/featurebase/v2/pql"
 	pb "github.com/molecula/featurebase/v2/proto"
 
@@ -326,6 +328,69 @@ func pgWriteGroupCount(w pg.QueryResultWriter, counts *pilosa.GroupCounts) error
 	return nil
 }
 
+func pgWriteStmtRows(w pg.QueryResultWriter, rows *pilosa.StmtRows) error {
+	//TODO(twg) writeHeader
+	columns := rows.Columns()
+	//TODO (twg) types:=rows.Types()
+	headers := make([]pg.ColumnInfo, len(columns))
+	for i, column := range columns {
+		headers[i] = pg.ColumnInfo{
+			Name: column,
+			Type: pg.TypeCharoid, //TODO(twg) types[i]
+		}
+	}
+	err := w.WriteHeader(headers...)
+	if err != nil {
+		return err
+	}
+	//TODO(twg) writeColumns
+	data := make([]string, len(headers))
+	for rows.Next() {
+		result := make([]interface{}, len(rows.Columns()))
+		// Create list of scan destination pointers.
+		dsts := make([]interface{}, len(result))
+		for i := range result {
+			dsts[i] = &result[i]
+		}
+
+		if err := rows.Scan(dsts...); err != nil {
+			return err
+		}
+		//TODO(twg) conversion should be happening in Scan as described in https://pkg.go.dev/database/sql
+		// ....
+		// Scan also converts between string and numeric types, as long as no information
+		// would be lost. While Scan stringifies all numbers scanned from numeric database columns into *string,
+		// scans into numeric types are checked for overflow. For example, a float64 with value 300 or a string
+		// with value "300" can scan into a uint16, but not into a uint8, though float64(255) or "255" can scan
+		// into a uint8. One exception is that scans of some float64 numbers to strings may lose information when stringifying.
+		// In general, scan floating point columns into *float64.
+		// ...
+		for i, col := range dsts {
+			var v string
+			switch col := col.(type) {
+			case nil:
+				v = "null"
+				//
+				//v = strconv.FormatUint(col.Uint64Val, 10)
+			default:
+				return errors.Errorf("unable to process value of type %T", col)
+			}
+
+			data[i] = v
+		}
+
+		err = w.WriteRowText(data...)
+		if err != nil {
+			return err
+		}
+
+	}
+	if err := rows.Err(); err != nil {
+		return err
+	}
+	return nil
+}
+
 func pgWriteRowser(w pg.QueryResultWriter, result pb.ToRowser) error {
 	var data []string
 	return result.ToRows(func(row *pb.RowResponse) error {
@@ -399,6 +464,8 @@ func pgWriteResult(w pg.QueryResultWriter, result interface{}) error {
 		return pgWriteGroupCount(w, result)
 	case pb.ToRowser: // we should avoid protobuf where we can...
 		return pgWriteRowser(w, result)
+	case *pilosa.StmtRows: // we should avoid protobuf where we can...
+		return pgWriteStmtRows(w, result)
 	case uint64:
 		err := w.WriteHeader(pg.ColumnInfo{
 			Name: "count",
@@ -469,11 +536,26 @@ func (pqh *PilosaQueryHandler) HandleQuery(ctx context.Context, w pg.QueryResult
 		return errors.Wrap(pgWriteResult(w, resp.Results[0]), "writing query result")
 
 	case pg.SimpleQuery:
-		resp, err := execSQL(ctx, pqh.Api, pqh.logger, string(q))
-		if err != nil {
-			return errors.Wrap(err, "executing query")
+		sql2 := false
+		if sql2 {
+			stmt, err := pqh.Api.Plan(ctx, string(q))
+			if err != nil {
+				return err
+			}
+			resp, err := stmt.QueryContext(ctx)
+			if err != nil {
+				return err
+			}
+			return errors.Wrap(pgWriteResult(w, resp), "writing sql2 query result")
+			//version 2.0
+		} else {
+			//version 1.0
+			resp, err := execSQL(ctx, pqh.Api, pqh.logger, string(q))
+			if err != nil {
+				return errors.Wrap(err, "executing query")
+			}
+			return errors.Wrap(pgWriteResult(w, resp), "writing query result")
 		}
-		return errors.Wrap(pgWriteResult(w, resp), "writing query result")
 
 	default:
 		return errors.Errorf("query type %T not yet supported (query: %s)", q, q)

From 9c8538ec5adc37a72839ac401328ff17ede4426c Mon Sep 17 00:00:00 2001
From: Todd Gruben <todd@pilosa.com>
Date: Fri, 17 Sep 2021 14:46:30 -0500
Subject: [PATCH 27/66] docker fix not really related to anything

---
 planner.go   |  5 +++++
 server.go    |  2 ++
 server/pg.go | 43 ++++++++++++++++++++++++++++---------------
 3 files changed, 35 insertions(+), 15 deletions(-)

diff --git a/planner.go b/planner.go
index ce89deebe..e60d628ef 100644
--- a/planner.go
+++ b/planner.go
@@ -23,6 +23,7 @@ import (
 
 	"github.com/molecula/featurebase/v2/pql"
 	"github.com/molecula/featurebase/v2/sql2"
+	"github.com/molecula/featurebase/v2/vprint"
 )
 
 type Planner struct {
@@ -645,6 +646,10 @@ func (rs *StmtRows) Columns() []*StmtColumn {
 	return rs.node.Columns()
 }
 
+func (rs *StmtRows) Row() int64 {
+	return rs.node.Row()[0].(int64)
+}
+
 func (rs *StmtRows) Next() bool {
 	if rs.err != nil {
 		return false
diff --git a/server.go b/server.go
index 93d9947a0..2e75045d4 100644
--- a/server.go
+++ b/server.go
@@ -39,6 +39,7 @@ import (
 	"github.com/molecula/featurebase/v2/stats"
 	"github.com/molecula/featurebase/v2/storage"
 	"github.com/molecula/featurebase/v2/topology"
+	"github.com/molecula/featurebase/v2/vprint"
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
 
@@ -1354,6 +1355,7 @@ func (s *Server) PlanSQL(ctx context.Context, q string) (*Stmt, error) {
 	if err != nil {
 		return nil, err
 	}
+	vprint.VV("PLanning SQL: (%v)", q)
 	return NewPlanner(s.executor).PlanStatement(ctx, st)
 }
 
diff --git a/server/pg.go b/server/pg.go
index a6395adff..7b8f39ad7 100644
--- a/server/pg.go
+++ b/server/pg.go
@@ -28,6 +28,7 @@ import (
 	pilosa "github.com/molecula/featurebase/v2"
 	"github.com/molecula/featurebase/v2/logger"
 	"github.com/molecula/featurebase/v2/pg"
+	"github.com/molecula/featurebase/v2/vprint"
 
 	//"github.com/molecula/featurebase/v2/pg"
 	"github.com/molecula/featurebase/v2/pql"
@@ -330,22 +331,32 @@ func pgWriteGroupCount(w pg.QueryResultWriter, counts *pilosa.GroupCounts) error
 
 func pgWriteStmtRows(w pg.QueryResultWriter, rows *pilosa.StmtRows) error {
 	//TODO(twg) writeHeader
-	columns := rows.Columns()
-	//TODO (twg) types:=rows.Types()
-	headers := make([]pg.ColumnInfo, len(columns))
-	for i, column := range columns {
-		headers[i] = pg.ColumnInfo{
-			Name: column,
-			Type: pg.TypeCharoid, //TODO(twg) types[i]
-		}
-	}
-	err := w.WriteHeader(headers...)
-	if err != nil {
-		return err
-	}
 	//TODO(twg) writeColumns
-	data := make([]string, len(headers))
+	first := true
+	var data []string
+	var err error
 	for rows.Next() {
+		if first {
+			columns := rows.Columns()
+			vprint.VV("ROW=> %#v", rows.Row())
+			//TODO (twg) types:=rows.Types()
+			headers := make([]pg.ColumnInfo, len(columns))
+			vprint.VV("got columns %v", columns)
+			for i, column := range columns {
+				headers[i] = pg.ColumnInfo{
+					Name: column,
+					Type: pg.TypeCharoid, //TODO(twg) types[i]
+				}
+			}
+			err := w.WriteHeader(headers...)
+			if err != nil {
+				return err
+			}
+
+			data = make([]string, len(headers))
+			first = false
+		}
+		vprint.VV("got row")
 		result := make([]interface{}, len(rows.Columns()))
 		// Create list of scan destination pointers.
 		dsts := make([]interface{}, len(result))
@@ -372,6 +383,8 @@ func pgWriteStmtRows(w pg.QueryResultWriter, rows *pilosa.StmtRows) error {
 				v = "null"
 				//
 				//v = strconv.FormatUint(col.Uint64Val, 10)
+			case *interface{}:
+				v = fmt.Sprintf("%v", *col)
 			default:
 				return errors.Errorf("unable to process value of type %T", col)
 			}
@@ -536,7 +549,7 @@ func (pqh *PilosaQueryHandler) HandleQuery(ctx context.Context, w pg.QueryResult
 		return errors.Wrap(pgWriteResult(w, resp.Results[0]), "writing query result")
 
 	case pg.SimpleQuery:
-		sql2 := false
+		sql2 := true
 		if sql2 {
 			stmt, err := pqh.Api.Plan(ctx, string(q))
 			if err != nil {

From 2ca7b107d2a0d21a052f49f5bf87d3f2ced2ba2d Mon Sep 17 00:00:00 2001
From: Todd Gruben <todd@pilosa.com>
Date: Fri, 17 Sep 2021 14:56:43 -0500
Subject: [PATCH 28/66] removed client1 from clustertests

---
 go.mod |  3 ++-
 go.sum | 26 ++++++++++++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/go.mod b/go.mod
index d64a32730..875729f63 100644
--- a/go.mod
+++ b/go.mod
@@ -29,6 +29,7 @@ require (
 	github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b
 	github.com/opentracing/opentracing-go v1.1.0
 	github.com/pelletier/go-toml v1.4.0
+	github.com/pilosa/pilosa/v2 v2.0.0-alpha.1
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.0.0
 	github.com/prometheus/client_model v0.1.0
@@ -54,7 +55,7 @@ require (
 	golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect
 	golang.org/x/text v0.3.5 // indirect
 	google.golang.org/grpc v1.28.0
-	gopkg.in/yaml.v2 v2.3.0
+	gopkg.in/yaml.v2 v2.3.0 // indirect
 	modernc.org/mathutil v1.0.0
 	modernc.org/strutil v1.0.0
 	sigs.k8s.io/yaml v1.2.0 // indirect
diff --git a/go.sum b/go.sum
index 5f4d1be61..a415da534 100644
--- a/go.sum
+++ b/go.sum
@@ -16,12 +16,14 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d h1:n0G4ckjMEj7bWuGYUX0i8YlBeBBJuZ+HEHvHfyBDZtI=
 github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d/go.mod h1:Rn2zM2MnHze07LwkneP48TWt6UiZhzQTwCvw6djVGfE=
+github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
 github.com/DataDog/datadog-go v2.2.0+incompatible h1:V5BKkxACZLjzHjSgBbr2gvLA2Ae49yhc6CSY7MLy5k4=
 github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
 github.com/HdrHistogram/hdrhistogram-go v1.1.0 h1:6dpdDPTRoo78HxAJ6T1HfMiKSnqhgRRqzCuPshRkQ7I=
 github.com/HdrHistogram/hdrhistogram-go v1.1.0/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo=
 github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
@@ -31,6 +33,7 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
+github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/beevik/ntp v0.3.0 h1:xzVrPrE4ziasFXgBVBZJDP0Wg/KpMwk2KHJ4Ba8GrDw=
@@ -42,6 +45,7 @@ github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
+github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
 github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
 github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@@ -51,8 +55,11 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
+github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
@@ -102,6 +109,7 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me
 github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M=
 github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
@@ -204,6 +212,7 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg=
 github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
@@ -244,6 +253,8 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.4.0 h1:u3Z1r+oOXJIkxqw34zVhyPgjBsm6X2wn21NWs/HfSeg=
 github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
+github.com/pilosa/pilosa/v2 v2.0.0-alpha.1 h1:dkKXzEtEvTHiJ7paskW6yqPBaUchOMj2VeXiM1QyEeQ=
+github.com/pilosa/pilosa/v2 v2.0.0-alpha.1/go.mod h1:CHRsL8ZpARQmEqiknUHFJSuEw9964DWCAdJQeT8vRK8=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -274,6 +285,7 @@ github.com/prometheus/prom2json v1.3.0/go.mod h1:rMN7m0ApCowcoDlypBHlkNbp5eJQf/+
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ=
 github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc=
+github.com/remyoudompheng/bigfft v0.0.0-20190321074620-2f0d2b0e0001/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 h1:HQagqIiBmr8YXawX/le3+O26N+vPPC1PtjaF3mwnook=
 github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
@@ -285,8 +297,11 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb
 github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM=
+github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/shirou/gopsutil/v3 v3.20.11 h1:NeVf1K0cgxsWz+N3671ojRptdgzvp7BXL3KV21R0JnA=
 github.com/shirou/gopsutil/v3 v3.20.11/go.mod h1:igHnfak0qnw1biGeI2qKQvu0ZkwvEkUcCLlYhZzdr/4=
+github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
@@ -312,6 +327,7 @@ github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.3.1/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
 github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk=
 github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
@@ -329,13 +345,18 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
+github.com/uber/jaeger-client-go v2.16.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
 github.com/uber/jaeger-client-go v2.25.0+incompatible h1:IxcNZ7WRY1Y3G4poYlx24szfsn/3LvK9QHCq9oQw8+U=
 github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
+github.com/uber/jaeger-lib v2.0.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
 github.com/uber/jaeger-lib v2.4.0+incompatible h1:fY7QsGQWiCt8pajv4r7JEvmATdCVaWxXbjwyYwsNaLQ=
 github.com/uber/jaeger-lib v2.4.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
+github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY=
@@ -358,7 +379,9 @@ go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -404,6 +427,7 @@ golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
@@ -432,10 +456,12 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

From 5176b7ff03e91e7486a792b277848dc8fa2d593c Mon Sep 17 00:00:00 2001
From: Todd Gruben <todd@pilosa.com>
Date: Tue, 21 Sep 2021 09:59:57 -0500
Subject: [PATCH 29/66] added sqlversion config option

---
 ctl/server.go    |  1 +
 go.mod           |  4 ++--
 pg/protocol.go   |  2 +-
 server/config.go |  3 +++
 server/pg.go     | 26 +++++++++++++++++---------
 server/server.go |  4 ++--
 6 files changed, 26 insertions(+), 14 deletions(-)

diff --git a/ctl/server.go b/ctl/server.go
index 2aa617295..6c4405135 100644
--- a/ctl/server.go
+++ b/ctl/server.go
@@ -111,6 +111,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
 	flags.DurationVar((*time.Duration)(&srv.Config.Postgres.WriteTimeout), "postgres.write-timeout", time.Duration(srv.Config.Postgres.WriteTimeout), "Timeout for writes on a postgres connection. (set 0 to disable)")
 	flags.Uint32Var(&srv.Config.Postgres.MaxStartupSize, "postgres.max-startup-size", srv.Config.Postgres.MaxStartupSize, "Maximum acceptable size of a postgres startup packet, in bytes. (set 0 to disable)")
 	flags.Uint16Var(&srv.Config.Postgres.ConnectionLimit, "postgres.connection-limit", srv.Config.Postgres.ConnectionLimit, "Maximum number of simultaneous postgres connections to allow. (set 0 to disable)")
+	flags.Uint16Var(&srv.Config.Postgres.SqlVersion, "postgres.sql-version", srv.Config.Postgres.SqlVersion, "Molecula Sql Handling Version (default 1)")
 
 	// Disk and Memory usage cache for ui/usage endpoint
 	flags.Float64Var(&srv.Config.UsageDutyCycle, "usage-duty-cycle", srv.Config.UsageDutyCycle, "Sets the percentage of time that is spent recalculating the disk and memory usage cache. 100.0 for always-running, 0 disables the cache and the /ui/usage endpoint.")
diff --git a/go.mod b/go.mod
index 875729f63..c99edd34d 100644
--- a/go.mod
+++ b/go.mod
@@ -29,7 +29,7 @@ require (
 	github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b
 	github.com/opentracing/opentracing-go v1.1.0
 	github.com/pelletier/go-toml v1.4.0
-	github.com/pilosa/pilosa/v2 v2.0.0-alpha.1
+	github.com/pilosa/pilosa/v2 v2.0.0-alpha.1 // indirect
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.0.0
 	github.com/prometheus/client_model v0.1.0
@@ -55,7 +55,7 @@ require (
 	golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect
 	golang.org/x/text v0.3.5 // indirect
 	google.golang.org/grpc v1.28.0
-	gopkg.in/yaml.v2 v2.3.0 // indirect
+	gopkg.in/yaml.v2 v2.3.0
 	modernc.org/mathutil v1.0.0
 	modernc.org/strutil v1.0.0
 	sigs.k8s.io/yaml v1.2.0 // indirect
diff --git a/pg/protocol.go b/pg/protocol.go
index 377e2ac16..041c331a4 100644
--- a/pg/protocol.go
+++ b/pg/protocol.go
@@ -1043,7 +1043,7 @@ func (s *Server) handleQuery(w message.Writer, query Query, cancelNotify <-chan
 		te:  s.TypeEngine,
 		tag: "SELECT",
 	}
-
+	vprint.VV("handle Query: (%v)", query)
 	// Dispatch the query handler.
 	qerr := s.QueryHandler.HandleQuery(ctx, qwriter, query)
 	if qerr != nil {
diff --git a/server/config.go b/server/config.go
index b96a5be1f..dc6ae08ae 100644
--- a/server/config.go
+++ b/server/config.go
@@ -200,6 +200,9 @@ type Config struct {
 		// Setting this to 0 disables the limit.
 		// This mostly exists because other DBs seem to have it.
 		ConnectionLimit uint16 `toml:"max-connections"`
+		// SqlVersion is which type of sqlhandling to be applied.
+		// The constant SqlV2 can be used to try the new experimental Molecula SQL handling
+		SqlVersion uint16 `toml:"sql-version"`
 	} `toml:"postgres"`
 
 	// Storage.Backend determines which Tx implementation the holder/Index will
diff --git a/server/pg.go b/server/pg.go
index 7b8f39ad7..4f59caaad 100644
--- a/server/pg.go
+++ b/server/pg.go
@@ -46,14 +46,20 @@ type PostgresServer struct {
 	s      pg.Server
 	stop   context.CancelFunc
 }
+type SqlVersion uint16
+
+const (
+	SqlV1 SqlVersion = 0
+	SqlV2 SqlVersion = 2
+)
 
 // NewPostgresServer creates a postgres server.
-func NewPostgresServer(api *pilosa.API, logger logger.Logger, tls *tls.Config) *PostgresServer {
+func NewPostgresServer(api *pilosa.API, logger logger.Logger, tls *tls.Config, sqlVersion SqlVersion) *PostgresServer {
 	return &PostgresServer{
 		api:    api,
 		logger: logger,
 		s: pg.Server{
-			QueryHandler:   NewPostgresHandler(api, logger),
+			QueryHandler:   NewPostgresHandler(api, logger, sqlVersion),
 			TypeEngine:     pg.PrimitiveTypeEngine{},
 			StartupTimeout: 5 * time.Second,
 			ReadTimeout:    10 * time.Second,
@@ -69,11 +75,12 @@ func NewPostgresServer(api *pilosa.API, logger logger.Logger, tls *tls.Config) *
 }
 
 // NewPostgresHandler creates a postgres query handler wrapping the pilosa API.
-func NewPostgresHandler(api *pilosa.API, logger logger.Logger) pg.QueryHandler {
+func NewPostgresHandler(api *pilosa.API, logger logger.Logger, sqlVersion SqlVersion) pg.QueryHandler {
 	return &QueryDecodeHandler{
 		Child: &PilosaQueryHandler{
-			Api:    api,
-			logger: logger,
+			Api:        api,
+			logger:     logger,
+			sqlVersion: sqlVersion,
 		},
 	}
 }
@@ -143,8 +150,9 @@ func pgDecodePQL(str string) (q pg.Query, err error) {
 }
 
 type PilosaQueryHandler struct {
-	Api    *pilosa.API
-	logger logger.Logger
+	Api        *pilosa.API
+	logger     logger.Logger
+	sqlVersion SqlVersion
 }
 
 func pgWriteRow(w pg.QueryResultWriter, row *pilosa.Row) error {
@@ -549,9 +557,9 @@ func (pqh *PilosaQueryHandler) HandleQuery(ctx context.Context, w pg.QueryResult
 		return errors.Wrap(pgWriteResult(w, resp.Results[0]), "writing query result")
 
 	case pg.SimpleQuery:
-		sql2 := true
-		if sql2 {
+		if pqh.sqlVersion == SqlV2 {
 			stmt, err := pqh.Api.Plan(ctx, string(q))
+			vprint.VV("SQL2Plan: (%v) (%v)", string(q), err)
 			if err != nil {
 				return err
 			}
diff --git a/server/server.go b/server/server.go
index 5a8478513..8d227ade7 100644
--- a/server/server.go
+++ b/server/server.go
@@ -40,7 +40,7 @@ import (
 
 	"golang.org/x/sync/errgroup"
 
-	"github.com/molecula/featurebase/v2"
+	pilosa "github.com/molecula/featurebase/v2"
 	"github.com/molecula/featurebase/v2/boltdb"
 	"github.com/molecula/featurebase/v2/encoding/proto"
 	petcd "github.com/molecula/featurebase/v2/etcd"
@@ -265,7 +265,7 @@ func (m *Command) Start() (err error) {
 			}
 			tlsConf = conf
 		}
-		m.pgserver = NewPostgresServer(m.API, m.logger, tlsConf)
+		m.pgserver = NewPostgresServer(m.API, m.logger, tlsConf, SqlVersion(m.Config.Postgres.SqlVersion))
 		m.pgserver.s.StartupTimeout = time.Duration(m.Config.Postgres.StartupTimeout)
 		m.pgserver.s.ReadTimeout = time.Duration(m.Config.Postgres.ReadTimeout)
 		m.pgserver.s.WriteTimeout = time.Duration(m.Config.Postgres.WriteTimeout)

From d6dd1e025bbef1663c81220c1940d9a080da0fb4 Mon Sep 17 00:00:00 2001
From: Todd Gruben <todd@pilosa.com>
Date: Wed, 22 Sep 2021 09:28:59 -0500
Subject: [PATCH 30/66] removed extra command complete message cleanup

---
 go.mod            |   3 +-
 go.sum            |  26 -------
 pg/protocol.go    | 174 ++++++++++++++++++++++------------------------
 server/pg_test.go |   4 +-
 4 files changed, 88 insertions(+), 119 deletions(-)

diff --git a/go.mod b/go.mod
index c99edd34d..e590bdd1b 100644
--- a/go.mod
+++ b/go.mod
@@ -29,7 +29,6 @@ require (
 	github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b
 	github.com/opentracing/opentracing-go v1.1.0
 	github.com/pelletier/go-toml v1.4.0
-	github.com/pilosa/pilosa/v2 v2.0.0-alpha.1 // indirect
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.0.0
 	github.com/prometheus/client_model v0.1.0
@@ -55,7 +54,7 @@ require (
 	golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect
 	golang.org/x/text v0.3.5 // indirect
 	google.golang.org/grpc v1.28.0
-	gopkg.in/yaml.v2 v2.3.0
+	gopkg.in/yaml.v2 v2.3.0 // indirect
 	modernc.org/mathutil v1.0.0
 	modernc.org/strutil v1.0.0
 	sigs.k8s.io/yaml v1.2.0 // indirect
diff --git a/go.sum b/go.sum
index a415da534..5f4d1be61 100644
--- a/go.sum
+++ b/go.sum
@@ -16,14 +16,12 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d h1:n0G4ckjMEj7bWuGYUX0i8YlBeBBJuZ+HEHvHfyBDZtI=
 github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d/go.mod h1:Rn2zM2MnHze07LwkneP48TWt6UiZhzQTwCvw6djVGfE=
-github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
 github.com/DataDog/datadog-go v2.2.0+incompatible h1:V5BKkxACZLjzHjSgBbr2gvLA2Ae49yhc6CSY7MLy5k4=
 github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
 github.com/HdrHistogram/hdrhistogram-go v1.1.0 h1:6dpdDPTRoo78HxAJ6T1HfMiKSnqhgRRqzCuPshRkQ7I=
 github.com/HdrHistogram/hdrhistogram-go v1.1.0/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo=
 github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
@@ -33,7 +31,6 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
-github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/beevik/ntp v0.3.0 h1:xzVrPrE4ziasFXgBVBZJDP0Wg/KpMwk2KHJ4Ba8GrDw=
@@ -45,7 +42,6 @@ github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
-github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
 github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
 github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@@ -55,11 +51,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
-github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
-github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
-github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
@@ -109,7 +102,6 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me
 github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M=
 github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
-github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
@@ -212,7 +204,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg=
 github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
-github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
@@ -253,8 +244,6 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.4.0 h1:u3Z1r+oOXJIkxqw34zVhyPgjBsm6X2wn21NWs/HfSeg=
 github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
-github.com/pilosa/pilosa/v2 v2.0.0-alpha.1 h1:dkKXzEtEvTHiJ7paskW6yqPBaUchOMj2VeXiM1QyEeQ=
-github.com/pilosa/pilosa/v2 v2.0.0-alpha.1/go.mod h1:CHRsL8ZpARQmEqiknUHFJSuEw9964DWCAdJQeT8vRK8=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -285,7 +274,6 @@ github.com/prometheus/prom2json v1.3.0/go.mod h1:rMN7m0ApCowcoDlypBHlkNbp5eJQf/+
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ=
 github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc=
-github.com/remyoudompheng/bigfft v0.0.0-20190321074620-2f0d2b0e0001/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 h1:HQagqIiBmr8YXawX/le3+O26N+vPPC1PtjaF3mwnook=
 github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
@@ -297,11 +285,8 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb
 github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
-github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM=
-github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/shirou/gopsutil/v3 v3.20.11 h1:NeVf1K0cgxsWz+N3671ojRptdgzvp7BXL3KV21R0JnA=
 github.com/shirou/gopsutil/v3 v3.20.11/go.mod h1:igHnfak0qnw1biGeI2qKQvu0ZkwvEkUcCLlYhZzdr/4=
-github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
@@ -327,7 +312,6 @@ github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/spf13/viper v1.3.1/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
 github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk=
 github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
@@ -345,18 +329,13 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
-github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
-github.com/uber/jaeger-client-go v2.16.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
 github.com/uber/jaeger-client-go v2.25.0+incompatible h1:IxcNZ7WRY1Y3G4poYlx24szfsn/3LvK9QHCq9oQw8+U=
 github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
-github.com/uber/jaeger-lib v2.0.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
 github.com/uber/jaeger-lib v2.4.0+incompatible h1:fY7QsGQWiCt8pajv4r7JEvmATdCVaWxXbjwyYwsNaLQ=
 github.com/uber/jaeger-lib v2.4.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
-github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
-github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY=
@@ -379,9 +358,7 @@ go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -427,7 +404,6 @@ golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
@@ -456,12 +432,10 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
diff --git a/pg/protocol.go b/pg/protocol.go
index 041c331a4..29dd62638 100644
--- a/pg/protocol.go
+++ b/pg/protocol.go
@@ -258,8 +258,10 @@ type Result struct {
 }
 type PgType byte
 
+// Constants used to indicated query interception
+// Only pgPassOn is allowed to be processed in Featurebase query handling
 const (
-	pgNotPg          PgType = 'x'
+	pgPassOn         PgType = 'x'
 	pgBackendPid     PgType = 'a'
 	pgVersion        PgType = 'b'
 	pgCountType      PgType = 'c'
@@ -291,7 +293,7 @@ func (p *Portal) Reset() {
 	vprint.VV("Portal Reset")
 	p.Name = ""
 	p.sql = ""
-	p.pgspecial = pgNotPg
+	p.pgspecial = pgPassOn
 	p.commands = p.commands[:0]
 }
 
@@ -335,6 +337,7 @@ func (p *Portal) Parse(data []byte) {
 				case sql.SQLTypeSet:
 					p.Name = "SET"
 					set := query.Statement.(*sqlparser.Set)
+					p.pgspecial = 0
 					for _, item := range set.Exprs {
 						vprint.VV("SET name=(%v) ", item.Name)
 						if item.Name.String() == "application_name" {
@@ -348,7 +351,7 @@ func (p *Portal) Parse(data []byte) {
 					}
 				case sql.SQLTypeSelect:
 					p.Name = "SELECT"
-					p.pgspecial = pgNotPg
+					p.pgspecial = pgPassOn
 					stmt := query.Statement.(*sqlparser.Select)
 					for _, item := range stmt.SelectExprs {
 						switch expr := item.(type) {
@@ -360,12 +363,9 @@ func (p *Portal) Parse(data []byte) {
 								case "pg_backend_pid":
 									//SELECT pg_backend_pid()
 									p.pgspecial = pgBackendPid
-									//p.encoder.RowDescription() //TODO(twg) move to the right spt
 								case "pg_terminate_backend":
-									//need the arg 100
 									//select pg_terminate_backend(100)
 									vprint.VV("terminate %#v", colExpr.Exprs)
-									//colExpr.Exprs[0]
 									p.pgspecial = pgTerminate
 								case "version":
 									//SELECT VERSION() AS version
@@ -395,20 +395,15 @@ func (p *Portal) Parse(data []byte) {
 					}
 					p.sql = string(b)
 				case sql.SQLTypeBegin:
+					// Ignore BEGIN
 					p.pgspecial = pgBegin
-					//				panic("TODO")
-					// ingnore for now
 				case sql.SQLTypeShow:
 					p.Name = "SHOW"
-					p.pgspecial = pgNotPg
+					p.pgspecial = pgPassOn
 					p.sql = string(b)
 				}
 			}
 		}
-		// TODO (twg) hack parsing for short term
-		//	parts := strings.Split(strings.TrimSuffix(string(data), "\x00"), " ")
-		//	p.Name = parts[0][1:] //TODO (twg) major hack
-		vprint.VV("PG SET:(%v)", p.pgspecial)
 	} else {
 		vprint.VV("SETTING EMPTY")
 		p.pgspecial = pgEmpty
@@ -428,14 +423,15 @@ func (p *Portal) Describe() {
 	*/
 
 }
-func (p *Portal) Execute() (shouldTerminate bool, queryReady bool) {
+func (p *Portal) Execute() (shouldTerminate bool, queryReady bool, err error) {
 	queryReady = true
 	vprint.VV("port.Execute: (%v) q=(%v)", p.pgspecial, p.sql)
 	switch p.pgspecial {
 	case pgBackendPid:
-		rowDescription, err := p.Encoder.EncodeColumn("pg_backend_pid", int32(23), 4)
-		if err != nil {
-			panic(err)
+		rowDescription, e := p.Encoder.EncodeColumn("pg_backend_pid", int32(23), 4)
+		if e != nil {
+			err = e
+			return
 		}
 		p.Add(rowDescription)
 		pid := fmt.Sprintf("%v", p.pid)
@@ -443,31 +439,30 @@ func (p *Portal) Execute() (shouldTerminate bool, queryReady bool) {
 		p.Add(dataRow)
 		//needs data row with cancel token
 	case pgVersion:
-		rowDescription, err := p.Encoder.EncodeColumn("version", int32(25), -1)
-		if err != nil {
-			panic(err)
+		rowDescription, e := p.Encoder.EncodeColumn("version", int32(25), -1)
+		if e != nil {
+			err = e
+			return
 		}
 		p.Add(rowDescription)
 		dataRow, _ := p.Encoder.TextRow("PostgresSQL 13.0 (molecula)")
 		p.Add(dataRow)
 	case pgSelect1:
-		rowDescription, err := p.Encoder.EncodeColumn("?column?", int32(23), 4)
-		if err != nil {
-			panic(err)
+		rowDescription, e := p.Encoder.EncodeColumn("?column?", int32(23), 4)
+		if e != nil {
+			err = e
+			return
 		}
 		p.Add(rowDescription)
 		dataRow, _ := p.Encoder.TextRow("1")
 		p.Add(dataRow)
 	case pgCountType:
 		//need to block
-		vprint.VV("blocking til terminated")
 		<-p.server.lookerChannel
-		vprint.VV("Released")
-		//now need to send ParseComplete,BindComplete,RowDescription,Error(Terminate)
-		//TODO(twg) handle the error
-		rowDescription, err := p.Encoder.EncodeColumn("count", int32(20), 8)
+		rowDescription, e := p.Encoder.EncodeColumn("count", int32(20), 8)
 		if err != nil {
-			panic(err)
+			err = e
+			return
 		}
 		p.Add(rowDescription)
 		errorResponse, _ := p.Encoder.Error(
@@ -485,52 +480,68 @@ func (p *Portal) Execute() (shouldTerminate bool, queryReady bool) {
 			},
 		)
 		p.Add(errorResponse)
-		p.Sync() //send and Reset
-		return true, queryReady
+		e = p.Sync() //send and Reset
+
+		if e != nil {
+			err = e
+			return
+		}
+		return true, queryReady, nil
 	case pgQueryTime:
 		// need to return something so that the id can be queried
 		//need to return  SELECT pid as id, query as stmt, EXTRACT(seconds from query_start - NOW()) as elapsed_time          FROM pg_stat_activity
 		//seems like we need a map of pids to querys
-		err := p.server.dumpPortalsTo(p)
-		if err != nil {
+		e := p.server.dumpPortalsTo(p)
+		if e != nil {
+			err = e
 			return
 		}
 
 	case pgTerminate:
 		//note just have 1 lock that blocks all who try to count the activities
-		vprint.VV("removing the block ")
 		//TODO (twg) lock this
 		close(p.server.lookerChannel)                //release all the other blockers and allow them to terminate
 		p.server.lookerChannel = make(chan struct{}) //create a new one just in case
 		// i think it needs to return boolean true
-		rowDescription, err := p.Encoder.EncodeColumn("pg_terminate_backend", int32(16), 1)
-		if err != nil {
-			panic(err)
+		rowDescription, e := p.Encoder.EncodeColumn("pg_terminate_backend", int32(16), 1)
+		if e != nil {
+			err = e
+			return
 		}
 		p.Add(rowDescription)
 		dataRow, _ := p.Encoder.TextRow("t")
 		p.Add(dataRow)
-		commandComplete, _ := p.Encoder.CommandComplete("SELECT 1")
+		commandComplete, e := p.Encoder.CommandComplete("SELECT 1")
+		if e != nil {
+			err = e
+			return
+		}
 		p.Add(commandComplete)
-		p.Sync()
-		return false, queryReady
+		e = p.Sync()
+		if e != nil {
+			err = e
+			return
+		}
+		return false, queryReady, nil
 	case pgEmpty:
 		p.Add(message.NoData)
 		p.Add(message.EmptyQueryResponse)
-		p.Sync()
+		e := p.Sync()
+		if e != nil {
+			err = e
+			return
+		}
 		queryReady = true
-		return false, queryReady
+		return false, queryReady, nil
 	case pgSetApplication:
 		//needs to add/send status
-		msg, err := p.Encoder.ParameterStatus("application_name", "PostgreSQL JDBC Driver") //TODO (twg) should be saved from parse
-		if err != nil {
+		msg, e := p.Encoder.ParameterStatus("application_name", "PostgreSQL JDBC Driver")
+		if e != nil {
+			err = e
 			return
 		}
 		p.Add(msg)
 	case pgSchema:
-		//need to add the descrition for the 3 fields
-		//	---[Field 01]---  name='table_schema'  type=19  type_len=64  type_mod=4294967295  relid=13276  attnum=2  format=0
-		// ---[Field 02]---  name='table_name'  type=19  type_len=64  type_mod=4294967295  relid=13276  attnum=3  format=0
 		parts := []message.SimpleColumn{
 			{
 				Name:    "table_schema",
@@ -543,27 +554,28 @@ func (p *Portal) Execute() (shouldTerminate bool, queryReady bool) {
 				Typelen: 64,
 			},
 		}
-		rowDescription, err := p.Encoder.EncodeColumns(parts...)
-		if err != nil { //TODO (twg) shadowing err
+		rowDescription, e := p.Encoder.EncodeColumns(parts...)
+		if e != nil {
+			err = e
 			return
 		}
 		p.Add(rowDescription)
-		err = p.HandleSchema()
-		if err != nil {
+		e = p.HandleSchema()
+		if e != nil {
+			err = e
 			return
 		}
-		vprint.VV("SHOULd make schema")
 
-	case pgNotPg:
-		vprint.VV("GOOOO>(%v)", p.sql)
+	case pgPassOn:
 		query := SimpleQuery(p.sql)
-		err := p.server.handleQuery(p, query, p.cancelNotify)
-		if err != nil {
+		e := p.server.handleQuery(p, query, p.cancelNotify)
+		if e != nil {
+			err = e
 			return
 		}
+		return
 
 	case pgBegin:
-		vprint.VV("BEGIN")
 		p.Name = "BEGIN"
 	}
 
@@ -571,25 +583,23 @@ func (p *Portal) Execute() (shouldTerminate bool, queryReady bool) {
 	if len(p.Name) > 0 { //only send command complete for those that have names
 		vprint.VV("EXECING NAME: '%v'", p.Name)
 		message, _ := p.Encoder.CommandComplete(p.Name)
-		vprint.VV("AFTER")
 		p.Add(message)
 	}
 	return
 }
 
 // handleStandard handles a connection in the standard postgres wire protocol.
-func (p *Portal) Sync() {
+func (p *Portal) Sync() error {
 	for _, m := range p.commands {
 		vprint.VV("Sending: '%v'", m.Type)
 		err := p.Writer.WriteMessage(m)
 		if err != nil {
-			//TODO (twg) need to change signature to return error
-			vprint.VV("error %v", err)
-			return
+			return err
 		}
 	}
 	p.Writer.Flush()
 	p.Reset()
+	return nil
 }
 func (p *Portal) Add(m message.Message) {
 	cp := message.Message{Type: m.Type, Data: make([]byte, len(m.Data))}
@@ -607,7 +617,6 @@ func (p *Portal) HandleSchema() error {
 	return p.server.QueryHandler.HandleSchema(context.Background(), p)
 }
 
-//implement the QueryResultWriterqueryResultR
 func (p *Portal) WriteMessage(m message.Message) error {
 	p.Add(m)
 	return nil
@@ -637,7 +646,7 @@ func (s *Server) handleStandard(ctx context.Context, proto Protocol, conn net.Co
 	if err != nil {
 		return errors.Wrap(err, "parsing parameters")
 	}
-	fmt.Printf("postgres connection params\n%#v\n", params) //TODO (twg) remove
+	vprint.VV("postgres connection params\n%#v\n", params) //TODO (twg) remove
 	if user, ok := params["user"]; ok {
 		// Log the connection.
 		s.Logger.Debugf("new postgres connection from user %q at %v", user, conn.RemoteAddr())
@@ -647,7 +656,6 @@ func (s *Server) handleStandard(ctx context.Context, proto Protocol, conn net.Co
 	}
 
 	// Set up message input and output.
-
 	// Set up a reader that will preempt the connection when the context is canceled.
 	ir := idleReader{
 		conn:    conn,
@@ -790,8 +798,6 @@ func (s *Server) handleStandard(ctx context.Context, proto Protocol, conn net.Co
 	for {
 		if !queryReady {
 			// Indicate that we are ready for a query.
-			// TODO: provide a valid transaction state.
-			//msg, err := encoder.ReadyForQuery(message.TransactionStatusActive)
 			portal.sql = ""
 			msg, err := encoder.ReadyForQuery(message.TransactionStatusIdle)
 			if err != nil {
@@ -855,31 +861,30 @@ func (s *Server) handleStandard(ctx context.Context, proto Protocol, conn net.Co
 			return w.Flush()
 
 		case message.TypeParse:
-			fmt.Println("Type Parse", strings.Trim(string(msg.Data), "\x00"))
+			vprint.VV("Type Parse (%v)", strings.Trim(string(msg.Data), "\x00"))
 			portal.Parse(msg.Data)
 		case message.TypeBind:
-			fmt.Println("Type Bind", strings.Trim(string(msg.Data), "\x00"))
-			//TODO(twg) apply values to bindings in the future
-			// Parse the query message (a null-terminated string).
+			vprint.VV("Type Bind (%v)", strings.Trim(string(msg.Data), "\x00"))
 			portal.Bind()
 		case message.TypeExecute:
-			fmt.Println("Type Execute", strings.Trim(string(msg.Data), "\x00"))
-			//term, qr := portal.Execute()
-			term, qr := portal.Execute()
+			vprint.VV("Type Execute (%v)", strings.Trim(string(msg.Data), "\x00"))
+			term, qr, err := portal.Execute()
+			if err != nil {
+				return err
+			}
 			if term {
 				return w.Flush()
 			}
 			queryReady = qr
 		case message.TypeSync:
-			fmt.Println("Type Sync", strings.Trim(string(msg.Data), "\x00"))
-			portal.Sync()
+			vprint.VV("Type Sync", strings.Trim(string(msg.Data), "\x00"))
+			err := portal.Sync()
+			if err != nil {
+				return err
+			}
 			queryReady = false
 		case message.TypeSimpleQuery:
-			// Execute a simple query.
-
 			queryReady = false
-
-			// Parse the query message (a null-terminated string).
 			query := SimpleQuery(strings.TrimSuffix(string(msg.Data), "\x00"))
 
 			// Execute the query.
@@ -887,16 +892,7 @@ func (s *Server) handleStandard(ctx context.Context, proto Protocol, conn net.Co
 			if err != nil {
 				return err
 			}
-			label := "N/A"
-			switch msg.Data[0] {
-			case 'S': //prepared statement
-				label = "prepared statement"
-			case 'P': //portal
-				label = "portal"
-			}
-			vprint.VV("describe %v:'%v'", label, msg.Data[1:])
 		case message.TypeDescribe:
-			//TODO (twg) major hack alet
 			portal.Describe()
 		case message.TypeClose:
 			return w.Flush()
diff --git a/server/pg_test.go b/server/pg_test.go
index 012986afc..bd8a88521 100644
--- a/server/pg_test.go
+++ b/server/pg_test.go
@@ -21,7 +21,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/molecula/featurebase/v2"
+	pilosa "github.com/molecula/featurebase/v2"
 	"github.com/molecula/featurebase/v2/logger"
 	"github.com/molecula/featurebase/v2/pg"
 	"github.com/molecula/featurebase/v2/pg/pgtest"
@@ -33,7 +33,7 @@ func TestPostgresHandler(t *testing.T) {
 	m := test.RunCommand(t)
 	defer m.Close()
 
-	pgh := server.NewPostgresHandler(m.API, logger.NewLogfLogger(t))
+	pgh := server.NewPostgresHandler(m.API, logger.NewLogfLogger(t), server.SqlV1)
 
 	m.MustCreateIndex(t, "i", pilosa.IndexOptions{TrackExistence: true})
 	m.MustCreateField(t, "i", "set")

From 5bc1364cdb42eb6af97cb3a362760111ad91be75 Mon Sep 17 00:00:00 2001
From: Todd Gruben <todd@pilosa.com>
Date: Thu, 23 Sep 2021 11:05:05 -0500
Subject: [PATCH 31/66] wip

---
 pg/protocol.go | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/pg/protocol.go b/pg/protocol.go
index 29dd62638..3ea85ec1f 100644
--- a/pg/protocol.go
+++ b/pg/protocol.go
@@ -302,9 +302,16 @@ func (p *Portal) Bind() {
 }
 func (p *Portal) Parse(data []byte) {
 	p.queryStart = time.Now()
-	b := bytes.Trim(data, "\x00")
-	vprint.VV("PARSE RAW: (%v) (%d)", string(b), len(b))
-	if strings.Contains(string(b), "EXTRACT") {
+	queryStr := string(bytes.Trim(data, "\x00"))
+	vprint.VV("PARSE RAW: (%v) (%d)", queryStr, len(queryStr))
+	if strings.HasPrefix(queryStr, "[") {
+		p.sql = queryStr
+		p.Name = "PQL"
+		p.pgspecial = pgPassOn
+		p.Add(message.ParseOK)
+		return
+	}
+	if strings.Contains(queryStr, "EXTRACT") {
 		// had to add this hack because the vitis parser doesn't handle...
 		/*
 		  	SELECT pid as id,
@@ -315,14 +322,14 @@ func (p *Portal) Parse(data []byte) {
 		*/
 		p.pgspecial = pgQueryTime
 		p.Name = "SELECT"
-		p.sql = string(b)
+		p.sql = queryStr
 		p.Add(message.ParseOK)
 		return
 	}
 
-	if len(b) > 2 {
+	if len(queryStr) > 2 {
 
-		query, err := p.mapper.MapSQL(string(b))
+		query, err := p.mapper.MapSQL(queryStr)
 		if err != nil {
 			vprint.VV("Parse Err: '%v'", err)
 		} else {
@@ -393,14 +400,14 @@ func (p *Portal) Parse(data []byte) {
 							}
 						}
 					}
-					p.sql = string(b)
+					p.sql = queryStr
 				case sql.SQLTypeBegin:
 					// Ignore BEGIN
 					p.pgspecial = pgBegin
 				case sql.SQLTypeShow:
 					p.Name = "SHOW"
 					p.pgspecial = pgPassOn
-					p.sql = string(b)
+					p.sql = queryStr
 				}
 			}
 		}

From fc339b9a62b26ef0e6068d631163f1ac0010cb2f Mon Sep 17 00:00:00 2001
From: Todd Gruben <todd@pilosa.com>
Date: Thu, 23 Sep 2021 12:02:29 -0500
Subject: [PATCH 32/66] skipp looker comments on PQL

---
 pg/protocol.go | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/pg/protocol.go b/pg/protocol.go
index 3ea85ec1f..248ea6e77 100644
--- a/pg/protocol.go
+++ b/pg/protocol.go
@@ -25,6 +25,7 @@ import (
 	"io"
 	"io/ioutil"
 	"net"
+	"regexp"
 	"strings"
 	"sync"
 	"time"
@@ -304,8 +305,11 @@ func (p *Portal) Parse(data []byte) {
 	p.queryStart = time.Now()
 	queryStr := string(bytes.Trim(data, "\x00"))
 	vprint.VV("PARSE RAW: (%v) (%d)", queryStr, len(queryStr))
-	if strings.HasPrefix(queryStr, "[") {
-		p.sql = queryStr
+	lookPQL := regexp.MustCompile("\\[.*\\].*\\)\\z")
+	foundPQL := lookPQL.FindStringSubmatch(queryStr)
+	if len(foundPQL) > 0 {
+
+		p.sql = foundPQL[0]
 		p.Name = "PQL"
 		p.pgspecial = pgPassOn
 		p.Add(message.ParseOK)

From 3baf15226ee7b983962cb7f53fffac64a34c154a Mon Sep 17 00:00:00 2001
From: Todd Gruben <todd@pilosa.com>
Date: Thu, 23 Sep 2021 12:09:42 -0500
Subject: [PATCH 33/66] make lookPQL a package var

---
 pg/protocol.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/pg/protocol.go b/pg/protocol.go
index 248ea6e77..d0e896975 100644
--- a/pg/protocol.go
+++ b/pg/protocol.go
@@ -301,11 +301,13 @@ func (p *Portal) Reset() {
 func (p *Portal) Bind() {
 	p.Add(message.BindComplete)
 }
+
+var lookPQL = regexp.MustCompile(`\[.*\].*\)\z`)
+
 func (p *Portal) Parse(data []byte) {
 	p.queryStart = time.Now()
 	queryStr := string(bytes.Trim(data, "\x00"))
 	vprint.VV("PARSE RAW: (%v) (%d)", queryStr, len(queryStr))
-	lookPQL := regexp.MustCompile("\\[.*\\].*\\)\\z")
 	foundPQL := lookPQL.FindStringSubmatch(queryStr)
 	if len(foundPQL) > 0 {
 

From 2053319db0b17a065c3e80c3532a13129a539f01 Mon Sep 17 00:00:00 2001
From: Todd Gruben <todd@pilosa.com>
Date: Sat, 25 Sep 2021 13:50:10 -0500
Subject: [PATCH 34/66] silence reporting

---
 pg/protocol.go | 30 ++----------------------------
 1 file changed, 2 insertions(+), 28 deletions(-)

diff --git a/pg/protocol.go b/pg/protocol.go
index d0e896975..fdfce083e 100644
--- a/pg/protocol.go
+++ b/pg/protocol.go
@@ -32,7 +32,6 @@ import (
 
 	"github.com/molecula/featurebase/v2/pg/message"
 	"github.com/molecula/featurebase/v2/sql"
-	"github.com/molecula/featurebase/v2/vprint"
 	"github.com/pkg/errors"
 	"vitess.io/vitess/go/vt/sqlparser"
 )
@@ -177,7 +176,6 @@ startup:
 		// Reject the unsecured connection.
 		return errors.Errorf("client at %s attempted to initiate an unsecured postgres conenction", conn.RemoteAddr())
 	}
-	vprint.VV("TProcotcol: %v", proto)
 
 	switch proto {
 	case ProtocolCancel:
@@ -217,7 +215,6 @@ func parseParams(data []byte) (map[string]string, error) {
 
 // handleCancel handles cancel request connections.
 func (s *Server) handleCancel(ctx context.Context, conn net.Conn, data []byte) error {
-	vprint.VV("handle cancel")
 	if len(data) != 8 {
 		return errors.New("malformed cancellation packet")
 	}
@@ -229,7 +226,6 @@ func (s *Server) handleCancel(ctx context.Context, conn net.Conn, data []byte) e
 	pid := int32(binary.BigEndian.Uint32(data[:4]))
 	key := int32(binary.BigEndian.Uint32(data[4:]))
 
-	vprint.VV("cancel pid:%v key:%v", pid, key)
 	err := s.CancellationManager.Cancel(CancellationToken{PID: pid, Key: key})
 	switch err {
 	case nil:
@@ -291,7 +287,6 @@ type Portal struct {
 }
 
 func (p *Portal) Reset() {
-	vprint.VV("Portal Reset")
 	p.Name = ""
 	p.sql = ""
 	p.pgspecial = pgPassOn
@@ -307,7 +302,6 @@ var lookPQL = regexp.MustCompile(`\[.*\].*\)\z`)
 func (p *Portal) Parse(data []byte) {
 	p.queryStart = time.Now()
 	queryStr := string(bytes.Trim(data, "\x00"))
-	vprint.VV("PARSE RAW: (%v) (%d)", queryStr, len(queryStr))
 	foundPQL := lookPQL.FindStringSubmatch(queryStr)
 	if len(foundPQL) > 0 {
 
@@ -337,12 +331,10 @@ func (p *Portal) Parse(data []byte) {
 
 		query, err := p.mapper.MapSQL(queryStr)
 		if err != nil {
-			vprint.VV("Parse Err: '%v'", err)
+			return
 		} else {
 
-			vprint.VV("TODD Query: %#v", query.SQL)
 			if strings.Contains(strings.ToLower(query.SQL), "select 1") {
-				vprint.VV("SQL1")
 				p.pgspecial = pgSelect1
 				p.Name = "SELECT"
 			} else {
@@ -352,12 +344,9 @@ func (p *Portal) Parse(data []byte) {
 					set := query.Statement.(*sqlparser.Set)
 					p.pgspecial = 0
 					for _, item := range set.Exprs {
-						vprint.VV("SET name=(%v) ", item.Name)
 						if item.Name.String() == "application_name" {
-							vprint.VV("SET expr=(%#v) ", item.Expr)
-							switch val := item.Expr.(type) {
+							switch item.Expr.(type) {
 							case *sqlparser.SQLVal:
-								vprint.VV("getting %v (%v)", val.Type, string(val.Val))
 								p.pgspecial = pgSetApplication
 							}
 						}
@@ -378,7 +367,6 @@ func (p *Portal) Parse(data []byte) {
 									p.pgspecial = pgBackendPid
 								case "pg_terminate_backend":
 									//select pg_terminate_backend(100)
-									vprint.VV("terminate %#v", colExpr.Exprs)
 									p.pgspecial = pgTerminate
 								case "version":
 									//SELECT VERSION() AS version
@@ -395,7 +383,6 @@ func (p *Portal) Parse(data []byte) {
 						switch from := item.(type) {
 						case *sqlparser.AliasedTableExpr:
 							tableName := from.Expr.(sqlparser.TableName).ToViewName().Name.String()
-							vprint.VV("looking at (%v)", tableName)
 							switch tableName {
 							case "pg_type":
 								p.pgspecial = pgCountType
@@ -418,7 +405,6 @@ func (p *Portal) Parse(data []byte) {
 			}
 		}
 	} else {
-		vprint.VV("SETTING EMPTY")
 		p.pgspecial = pgEmpty
 	}
 	p.Add(message.ParseOK)
@@ -438,7 +424,6 @@ func (p *Portal) Describe() {
 }
 func (p *Portal) Execute() (shouldTerminate bool, queryReady bool, err error) {
 	queryReady = true
-	vprint.VV("port.Execute: (%v) q=(%v)", p.pgspecial, p.sql)
 	switch p.pgspecial {
 	case pgBackendPid:
 		rowDescription, e := p.Encoder.EncodeColumn("pg_backend_pid", int32(23), 4)
@@ -594,7 +579,6 @@ func (p *Portal) Execute() (shouldTerminate bool, queryReady bool, err error) {
 
 	//maybe add in the number of items in select clause
 	if len(p.Name) > 0 { //only send command complete for those that have names
-		vprint.VV("EXECING NAME: '%v'", p.Name)
 		message, _ := p.Encoder.CommandComplete(p.Name)
 		p.Add(message)
 	}
@@ -604,7 +588,6 @@ func (p *Portal) Execute() (shouldTerminate bool, queryReady bool, err error) {
 // handleStandard handles a connection in the standard postgres wire protocol.
 func (p *Portal) Sync() error {
 	for _, m := range p.commands {
-		vprint.VV("Sending: '%v'", m.Type)
 		err := p.Writer.WriteMessage(m)
 		if err != nil {
 			return err
@@ -659,7 +642,6 @@ func (s *Server) handleStandard(ctx context.Context, proto Protocol, conn net.Co
 	if err != nil {
 		return errors.Wrap(err, "parsing parameters")
 	}
-	vprint.VV("postgres connection params\n%#v\n", params) //TODO (twg) remove
 	if user, ok := params["user"]; ok {
 		// Log the connection.
 		s.Logger.Debugf("new postgres connection from user %q at %v", user, conn.RemoteAddr())
@@ -773,7 +755,6 @@ func (s *Server) handleStandard(ctx context.Context, proto Protocol, conn net.Co
 		return errors.Wrap(err, "sending parameter status server version")
 	}
 
-	vprint.VV("CancellationManger: %v", s.CancellationManager)
 	var cancelNotify <-chan struct{}
 	var pid int32
 	if s.CancellationManager != nil {
@@ -874,13 +855,10 @@ func (s *Server) handleStandard(ctx context.Context, proto Protocol, conn net.Co
 			return w.Flush()
 
 		case message.TypeParse:
-			vprint.VV("Type Parse (%v)", strings.Trim(string(msg.Data), "\x00"))
 			portal.Parse(msg.Data)
 		case message.TypeBind:
-			vprint.VV("Type Bind (%v)", strings.Trim(string(msg.Data), "\x00"))
 			portal.Bind()
 		case message.TypeExecute:
-			vprint.VV("Type Execute (%v)", strings.Trim(string(msg.Data), "\x00"))
 			term, qr, err := portal.Execute()
 			if err != nil {
 				return err
@@ -890,7 +868,6 @@ func (s *Server) handleStandard(ctx context.Context, proto Protocol, conn net.Co
 			}
 			queryReady = qr
 		case message.TypeSync:
-			vprint.VV("Type Sync", strings.Trim(string(msg.Data), "\x00"))
 			err := portal.Sync()
 			if err != nil {
 				return err
@@ -912,7 +889,6 @@ func (s *Server) handleStandard(ctx context.Context, proto Protocol, conn net.Co
 		default:
 			// The message is not supported yet.
 			// Send an error.
-			vprint.VV("unrecognized postgres packet %v", msg)
 			s.Logger.Errorf("unrecognized postgres packet %v", msg)
 			msg, err = encoder.Error(
 				message.NoticeField{
@@ -992,7 +968,6 @@ func (s *Server) dumpPortalsTo(p *Portal) error {
 	p.Add(rowDescription)
 
 	for _, portal := range s.portals {
-		vprint.VV("port: %v (%v) %v", portal.pid, portal.sql, time.Since(portal.queryStart))
 		dataRow, err := p.Encoder.TextRow(
 			fmt.Sprintf("%v", portal.pid),
 			portal.sql,
@@ -1052,7 +1027,6 @@ func (s *Server) handleQuery(w message.Writer, query Query, cancelNotify <-chan
 		te:  s.TypeEngine,
 		tag: "SELECT",
 	}
-	vprint.VV("handle Query: (%v)", query)
 	// Dispatch the query handler.
 	qerr := s.QueryHandler.HandleQuery(ctx, qwriter, query)
 	if qerr != nil {

From c4e64528e035b68ab9e957b501c5c24eab7ea8df Mon Sep 17 00:00:00 2001
From: Todd Gruben <todd@pilosa.com>
Date: Sat, 25 Sep 2021 13:56:36 -0500
Subject: [PATCH 35/66] fix

---
 planner.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/planner.go b/planner.go
index e60d628ef..985a92001 100644
--- a/planner.go
+++ b/planner.go
@@ -23,7 +23,6 @@ import (
 
 	"github.com/molecula/featurebase/v2/pql"
 	"github.com/molecula/featurebase/v2/sql2"
-	"github.com/molecula/featurebase/v2/vprint"
 )
 
 type Planner struct {

From 0920c4c029acde5c96b3e966a8df922242a10ce1 Mon Sep 17 00:00:00 2001
From: Todd Gruben <todd@pilosa.com>
Date: Sat, 25 Sep 2021 14:06:55 -0500
Subject: [PATCH 36/66] silence and rebase

---
 server/pg.go | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/server/pg.go b/server/pg.go
index 4f59caaad..8b7d6478f 100644
--- a/server/pg.go
+++ b/server/pg.go
@@ -28,7 +28,6 @@ import (
 	pilosa "github.com/molecula/featurebase/v2"
 	"github.com/molecula/featurebase/v2/logger"
 	"github.com/molecula/featurebase/v2/pg"
-	"github.com/molecula/featurebase/v2/vprint"
 
 	//"github.com/molecula/featurebase/v2/pg"
 	"github.com/molecula/featurebase/v2/pql"
@@ -346,13 +345,11 @@ func pgWriteStmtRows(w pg.QueryResultWriter, rows *pilosa.StmtRows) error {
 	for rows.Next() {
 		if first {
 			columns := rows.Columns()
-			vprint.VV("ROW=> %#v", rows.Row())
 			//TODO (twg) types:=rows.Types()
 			headers := make([]pg.ColumnInfo, len(columns))
-			vprint.VV("got columns %v", columns)
 			for i, column := range columns {
 				headers[i] = pg.ColumnInfo{
-					Name: column,
+					Name: column.Name,
 					Type: pg.TypeCharoid, //TODO(twg) types[i]
 				}
 			}
@@ -364,7 +361,6 @@ func pgWriteStmtRows(w pg.QueryResultWriter, rows *pilosa.StmtRows) error {
 			data = make([]string, len(headers))
 			first = false
 		}
-		vprint.VV("got row")
 		result := make([]interface{}, len(rows.Columns()))
 		// Create list of scan destination pointers.
 		dsts := make([]interface{}, len(result))
@@ -559,7 +555,6 @@ func (pqh *PilosaQueryHandler) HandleQuery(ctx context.Context, w pg.QueryResult
 	case pg.SimpleQuery:
 		if pqh.sqlVersion == SqlV2 {
 			stmt, err := pqh.Api.Plan(ctx, string(q))
-			vprint.VV("SQL2Plan: (%v) (%v)", string(q), err)
 			if err != nil {
 				return err
 			}

From ed1cf7ffef85d92afb8aebd6f9efad2375a1ce4d Mon Sep 17 00:00:00 2001
From: Todd Gruben <todd@molecula.com>
Date: Mon, 27 Sep 2021 07:05:08 -0500
Subject: [PATCH 37/66] cleanup and applied review suggestions

---
 go.mod                |   2 +-
 pg/message/io.go      |   2 -
 pg/message/message.go |   6 +-
 pg/protocol.go        | 156 +++++++++++++++++++-----------------------
 planner.go            |   3 +-
 5 files changed, 78 insertions(+), 91 deletions(-)

diff --git a/go.mod b/go.mod
index e590bdd1b..d64a32730 100644
--- a/go.mod
+++ b/go.mod
@@ -54,7 +54,7 @@ require (
 	golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect
 	golang.org/x/text v0.3.5 // indirect
 	google.golang.org/grpc v1.28.0
-	gopkg.in/yaml.v2 v2.3.0 // indirect
+	gopkg.in/yaml.v2 v2.3.0
 	modernc.org/mathutil v1.0.0
 	modernc.org/strutil v1.0.0
 	sigs.k8s.io/yaml v1.2.0 // indirect
diff --git a/pg/message/io.go b/pg/message/io.go
index ec2321675..e09150b9d 100644
--- a/pg/message/io.go
+++ b/pg/message/io.go
@@ -18,7 +18,6 @@ import (
 	"bufio"
 	"encoding/binary"
 	"errors"
-	"fmt"
 	"io"
 )
 
@@ -93,7 +92,6 @@ type WireWriter struct {
 
 // WriteMessage writes a message onto the wire.
 func (w *WireWriter) WriteMessage(message Message) error {
-	fmt.Printf("SendToClient %c (%d)\n", message.Type, len(message.Data))
 	if uint(len(message.Data))+4 >= 1<<31 {
 		return ErrMessageTooBig
 	}
diff --git a/pg/message/message.go b/pg/message/message.go
index 757053297..9fb6871ee 100644
--- a/pg/message/message.go
+++ b/pg/message/message.go
@@ -30,7 +30,7 @@ const (
 	// TypeReadyForQuery is a message used to indicate that the server is ready for another query.
 	TypeReadyForQuery Type = 'Z'
 
-	// TypeCommandComplete is a message used to indicate that a query has completed. Backend
+	// TypeCommandComplete is a Backend message used to indicate that a query has completed.
 	TypeCommandComplete Type = 'C'
 
 	// TypeClos is a message used to indicate that a query has completed. Frontend
@@ -60,8 +60,8 @@ const (
 	TypeBind         Type = 'B'
 	TypeBindComplete Type = '2'
 
-	TypeExecute            Type = 'E' // Frontend TODO(TWG)
-	TypeError              Type = 'E' // Backend   SOMETHING NOT RIGHT HERE
+	TypeExecute            Type = 'E' // Frontend
+	TypeError              Type = 'E' // Backend
 	TypeSync               Type = 'S' // Frontend
 	TypeParameterStatus    Type = 'S' // Backend
 	TypeDescribe           Type = 'D' // Frontend
diff --git a/pg/protocol.go b/pg/protocol.go
index fdfce083e..0fa52a5e9 100644
--- a/pg/protocol.go
+++ b/pg/protocol.go
@@ -272,12 +272,11 @@ const (
 )
 
 type Portal struct {
-	Name     string
-	Writer   *message.WireWriter
-	commands []message.Message
-	Encoder  *message.Encoder
-	mapper   *sql.Mapper
-	//results      []Result
+	Name         string
+	Writer       *message.WireWriter
+	commands     []message.Message
+	Encoder      *message.Encoder
+	mapper       *sql.Mapper
 	sql          string
 	pgspecial    PgType
 	pid          int32
@@ -332,76 +331,75 @@ func (p *Portal) Parse(data []byte) {
 		query, err := p.mapper.MapSQL(queryStr)
 		if err != nil {
 			return
+		}
+
+		if strings.Contains(strings.ToLower(query.SQL), "select 1") {
+			p.pgspecial = pgSelect1
+			p.Name = "SELECT"
 		} else {
-
-			if strings.Contains(strings.ToLower(query.SQL), "select 1") {
-				p.pgspecial = pgSelect1
-				p.Name = "SELECT"
-			} else {
-				switch query.SQLType {
-				case sql.SQLTypeSet:
-					p.Name = "SET"
-					set := query.Statement.(*sqlparser.Set)
-					p.pgspecial = 0
-					for _, item := range set.Exprs {
-						if item.Name.String() == "application_name" {
-							switch item.Expr.(type) {
-							case *sqlparser.SQLVal:
-								p.pgspecial = pgSetApplication
-							}
+			switch query.SQLType {
+			case sql.SQLTypeSet:
+				p.Name = "SET"
+				set := query.Statement.(*sqlparser.Set)
+				p.pgspecial = 0
+				for _, item := range set.Exprs {
+					if item.Name.String() == "application_name" {
+						switch item.Expr.(type) {
+						case *sqlparser.SQLVal:
+							p.pgspecial = pgSetApplication
 						}
 					}
-				case sql.SQLTypeSelect:
-					p.Name = "SELECT"
-					p.pgspecial = pgPassOn
-					stmt := query.Statement.(*sqlparser.Select)
-					for _, item := range stmt.SelectExprs {
-						switch expr := item.(type) {
-						case *sqlparser.AliasedExpr:
-							switch colExpr := expr.Expr.(type) {
-							case *sqlparser.FuncExpr:
-								funcName := strings.ToLower(colExpr.Name.String())
-								switch funcName {
-								case "pg_backend_pid":
-									//SELECT pg_backend_pid()
-									p.pgspecial = pgBackendPid
-								case "pg_terminate_backend":
-									//select pg_terminate_backend(100)
-									p.pgspecial = pgTerminate
-								case "version":
-									//SELECT VERSION() AS version
-									p.pgspecial = pgVersion
-								}
-								//need to return  the pid from the cancelation object
-								//add row description object
-								//add data row for item
-							}
-						}
-
-					}
-					for _, item := range stmt.From {
-						switch from := item.(type) {
-						case *sqlparser.AliasedTableExpr:
-							tableName := from.Expr.(sqlparser.TableName).ToViewName().Name.String()
-							switch tableName {
-							case "pg_type":
-								p.pgspecial = pgCountType
-							case "pg_stat_activity":
-								p.pgspecial = pgQueryTime
-							case "tables":
-								p.pgspecial = pgSchema
-							}
-						}
-					}
-					p.sql = queryStr
-				case sql.SQLTypeBegin:
-					// Ignore BEGIN
-					p.pgspecial = pgBegin
-				case sql.SQLTypeShow:
-					p.Name = "SHOW"
-					p.pgspecial = pgPassOn
-					p.sql = queryStr
 				}
+			case sql.SQLTypeSelect:
+				p.Name = "SELECT"
+				p.pgspecial = pgPassOn
+				stmt := query.Statement.(*sqlparser.Select)
+				for _, item := range stmt.SelectExprs {
+					switch expr := item.(type) {
+					case *sqlparser.AliasedExpr:
+						switch colExpr := expr.Expr.(type) {
+						case *sqlparser.FuncExpr:
+							funcName := strings.ToLower(colExpr.Name.String())
+							switch funcName {
+							case "pg_backend_pid":
+								//SELECT pg_backend_pid()
+								p.pgspecial = pgBackendPid
+							case "pg_terminate_backend":
+								//select pg_terminate_backend(100)
+								p.pgspecial = pgTerminate
+							case "version":
+								//SELECT VERSION() AS version
+								p.pgspecial = pgVersion
+							}
+							//need to return  the pid from the cancelation object
+							//add row description object
+							//add data row for item
+						}
+					}
+
+				}
+				for _, item := range stmt.From {
+					switch from := item.(type) {
+					case *sqlparser.AliasedTableExpr:
+						tableName := from.Expr.(sqlparser.TableName).ToViewName().Name.String()
+						switch tableName {
+						case "pg_type":
+							p.pgspecial = pgCountType
+						case "pg_stat_activity":
+							p.pgspecial = pgQueryTime
+						case "tables":
+							p.pgspecial = pgSchema
+						}
+					}
+				}
+				p.sql = queryStr
+			case sql.SQLTypeBegin:
+				// Ignore BEGIN
+				p.pgspecial = pgBegin
+			case sql.SQLTypeShow:
+				p.Name = "SHOW"
+				p.pgspecial = pgPassOn
+				p.sql = queryStr
 			}
 		}
 	} else {
@@ -410,18 +408,9 @@ func (p *Portal) Parse(data []byte) {
 	p.Add(message.ParseOK)
 }
 func (p *Portal) Describe() {
-	/*
-		if p.pgspecial != pgNotPg {
-			//do custom handling for setup
-		}
-		if len(p.results) == 0 {
-			p.Add(&message.NoData)
-			p.Add(&message.EmptyQueryResponse)
-			return
-		}
-	*/
-
+	// Placeholder should we need to handle the Decribe request
 }
+
 func (p *Portal) Execute() (shouldTerminate bool, queryReady bool, err error) {
 	queryReady = true
 	switch p.pgspecial {
@@ -824,7 +813,6 @@ func (s *Server) handleStandard(ctx context.Context, proto Protocol, conn net.Co
 
 		// Read the next packet.
 		msg, err := r.ReadMessage()
-		msg.Dump("start-") // TODO(twg) remove
 		if err != nil {
 			if err == errPreempted {
 				// The server is shutting down.
diff --git a/planner.go b/planner.go
index 985a92001..196bfa817 100644
--- a/planner.go
+++ b/planner.go
@@ -645,10 +645,11 @@ func (rs *StmtRows) Columns() []*StmtColumn {
 	return rs.node.Columns()
 }
 
+	/*
 func (rs *StmtRows) Row() int64 {
 	return rs.node.Row()[0].(int64)
 }
-
+	*/
 func (rs *StmtRows) Next() bool {
 	if rs.err != nil {
 		return false

From c2caa6397899eb60f844baf10dd3be2809e14a64 Mon Sep 17 00:00:00 2001
From: Todd Gruben <todd@molecula.com>
Date: Mon, 27 Sep 2021 08:20:03 -0500
Subject: [PATCH 38/66] tidy

---
 go.mod | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/go.mod b/go.mod
index d64a32730..e590bdd1b 100644
--- a/go.mod
+++ b/go.mod
@@ -54,7 +54,7 @@ require (
 	golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect
 	golang.org/x/text v0.3.5 // indirect
 	google.golang.org/grpc v1.28.0
-	gopkg.in/yaml.v2 v2.3.0
+	gopkg.in/yaml.v2 v2.3.0 // indirect
 	modernc.org/mathutil v1.0.0
 	modernc.org/strutil v1.0.0
 	sigs.k8s.io/yaml v1.2.0 // indirect

From 9f271467fb1c73a0d83d50e28439e59413b40c53 Mon Sep 17 00:00:00 2001
From: Seebs <seebs@molecula.com>
Date: Fri, 10 Sep 2021 12:10:22 -0500
Subject: [PATCH 39/66] ingest cluster support

We add endpoints and protobuf encode/decode to allow for sending
sharded requests over the wire in protobuf, so we can take our
sharded data and send it to other nodes if needed.

This is a squash of >15 other commits, so a bit of history
is relevant:

The Request type had FieldTypes in it because the field type
information was needed for sharding because sorting requires
that information. We change this around to make the external
sharding operation require the field types, and curry that
through the codec -- the codec is needed to tell the request
how it shards. (This is because the correct sorting order
varies by field type.) Requests (and ShardedRequests) no
longer have that table in them.

And then we hit a nasty bug in production and RCA showed
that our testing wasn't good enough and we need to be more
careful, and I discovered that test coverage in this package
was around 70%.

So, the other big thing here is coverage testing; in order to
make coverage testing viable and programmatically testable,
we have added the ability to render requests *back* to
JSON. This is not a great idea, but it does allow us to do
a lot of sanity-checking and verify that the encodings we're
using are consistent and correct.

This, plus some specific tests of decoding specific flawed
inputs, has caught a number of issues. Which are now fixed!

A lot of internal API surface got slightly changed, in ways
that make it simpler to work with. For instance, the
(*FieldOperation).TranslateUnsigned function doesn't really
need to exist; we can just have a non-method translate
function for unsigned and for signed, and use them based on
field type.

The stable translation hack used for testing had a bug that
could allow it to end up producing incorrect results if you
asked it to translate an ID first rather than exclusively
asking it to translate strings first, this has been
corrected. (This is a bug fix in code that was added
partway through creating this, but is tricky enough to
mention its own comment.)

Test coverage is now just over 90%, and a lot of what's left
is error-check returns that may well be actually unreachable
unless, say, the documentation for encoding/json is full of
lies. Which it probably is.
---
 api.go                   |  94 +++-
 client.go                |   6 +
 cluster.go               |  34 ++
 encoding/proto/proto.go  | 112 ++++-
 http/client.go           |  33 ++
 http/handler.go          |  63 ++-
 ingest/codec.go          | 746 +++++++++++++++++++++++++----
 ingest/codec_test.go     | 990 +++++++++++++++++++++++++++++++++------
 ingest/op.go             | 345 +++++++++-----
 ingest/op_test.go        | 116 ++++-
 ingest/translate.go      |  71 +++
 ingest/translate_test.go |  70 +++
 ingest/vec.go            |  57 ++-
 ingest/vec_test.go       | 114 +++++
 ingest_test.go           |  11 +-
 translate.go             |  34 ++
 16 files changed, 2495 insertions(+), 401 deletions(-)
 create mode 100644 ingest/translate_test.go
 create mode 100644 ingest/vec_test.go

diff --git a/api.go b/api.go
index ce8ffda33..95e2199bf 100644
--- a/api.go
+++ b/api.go
@@ -1826,6 +1826,37 @@ func (api *API) ImportValueWithTx(ctx context.Context, qcx *Qcx, req *ImportValu
 	return nil
 }
 
+// helper function: do the apply stuff for a known index with known fields
+func (api *API) ingestNodeOperationsForFields(ctx context.Context, qcx *Qcx, index *Index, knownFields map[string]*Field, req *ingest.ShardedRequest) error {
+	eg, ctx := errgroup.WithContext(ctx)
+	for shard, ops := range req.Ops {
+		// loop variable shadow capture is the go equivalent of man door hook hand
+		shard, ops := shard, ops
+		eg.Go(func() error {
+			return api.applyOperations(ctx, qcx, index, shard, knownFields, ops)
+		})
+	}
+	return eg.Wait()
+}
+
+// IngestNodeOperations handles protobuf-formatted data which does not need
+// key translation and is applicable to this specific node.
+func (api *API) IngestNodeOperations(ctx context.Context, qcx *Qcx, indexName string, req *ingest.ShardedRequest) error {
+	index := api.holder.Index(indexName)
+	if index == nil {
+		api.server.logger.Errorf("ingest: no such index %q", indexName)
+		return newNotFoundError(ErrIndexNotFound, indexName)
+	}
+	fields := index.Fields()
+	knownFields := map[string]*Field{}
+	for _, field := range fields {
+		knownFields[field.name] = field
+	}
+	return api.ingestNodeOperationsForFields(ctx, qcx, index, knownFields, req)
+}
+
+// IngestOperations handles JSON-formatted data which may need key translation
+// and may be for any or all nodes.
 func (api *API) IngestOperations(ctx context.Context, qcx *Qcx, indexName string, stream io.Reader) error {
 	span, _ := tracing.StartSpanFromContext(ctx, "API.IngestOperations")
 	defer span.Finish()
@@ -1841,34 +1872,32 @@ func (api *API) IngestOperations(ctx context.Context, qcx *Qcx, indexName string
 		return newNotFoundError(ErrIndexNotFound, indexName)
 	}
 	fields := index.Fields()
-	var lookup ingest.KeyLookupFunc
+	var indexKeys ingest.KeyTranslator
 	if index.Keys() {
-		lookup = func(keys ...string) (map[string]uint64, error) {
-			return api.cluster.createIndexKeys(ctx, indexName, keys...)
-		}
+		indexKeys = newIngestKeyTranslatorFromCluster(ctx, api.cluster, indexName)
 	}
-	codec, err := ingest.NewJSONCodec(lookup)
+	codec, err := ingest.NewJSONCodec(indexKeys)
 	if err != nil {
 		return errors.Wrap(err, "creating JSON codec")
 	}
 	knownFields := map[string]*Field{}
 	for _, field := range fields {
-		var lookup ingest.KeyLookupFunc
+		var keys ingest.KeyTranslator
 		if field.usesKeys {
-			lookup = field.translateStore.CreateKeys
+			keys = newIngestKeyTranslatorFromStore(field.translateStore)
 		}
 		knownFields[field.name] = field
 		switch field.Type() {
 		case "set":
-			if err = codec.AddSetField(field.name, lookup); err != nil {
+			if err = codec.AddSetField(field.name, keys); err != nil {
 				return fmt.Errorf("adding set field to codec: %w", err)
 			}
 		case "time":
-			if err = codec.AddTimeQuantumField(field.name, lookup); err != nil {
+			if err = codec.AddTimeQuantumField(field.name, keys); err != nil {
 				return fmt.Errorf("adding time quantum field to codec: %w", err)
 			}
 		case "mutex":
-			if err = codec.AddMutexField(field.name, lookup); err != nil {
+			if err = codec.AddMutexField(field.name, keys); err != nil {
 				return fmt.Errorf("adding mutex field to codec: %w", err)
 			}
 		case "bool":
@@ -1876,7 +1905,7 @@ func (api *API) IngestOperations(ctx context.Context, qcx *Qcx, indexName string
 				return fmt.Errorf("adding bool field to codec: %w", err)
 			}
 		case "int":
-			if err = codec.AddIntField(field.name, lookup); err != nil {
+			if err = codec.AddIntField(field.name, keys); err != nil {
 				return fmt.Errorf("adding int field to codec: %w", err)
 			}
 		case "decimal":
@@ -1896,17 +1925,44 @@ func (api *API) IngestOperations(ctx context.Context, qcx *Qcx, indexName string
 	if err != nil {
 		return errors.Wrap(err, "parsing input data")
 	}
-	sharded, err := req.ByShard()
+	sharded, err := codec.RequestByShard(req)
 	if err != nil {
 		return errors.Wrap(err, "sharding input data")
 	}
-	eg, ctx := errgroup.WithContext(ctx)
+	// now that we have this, let's assign the shards to nodes
+	snap := topology.NewClusterSnapshot(api.cluster.noder, api.cluster.Hasher, api.cluster.ReplicaN)
+	// oh hey an easy case: we're presumably the only node
+	if len(snap.Nodes) == 1 {
+		return api.ingestNodeOperationsForFields(ctx, qcx, index, knownFields, sharded)
+	}
+	// split up by fields in some way
+	byNode := make(map[string]*ingest.ShardedRequest)
 	for shard, ops := range sharded.Ops {
-		// loop variable shadow capture is the go equivalent of man door hook hand
-		shard, ops := shard, ops
-		eg.Go(func() error {
-			return api.applyOperations(ctx, qcx, index, shard, knownFields, ops)
-		})
+		nodes := snap.ShardNodes(indexName, shard)
+		forThisShard := byNode[nodes[0].ID]
+		if forThisShard == nil {
+			byNode[nodes[0].ID] = &ingest.ShardedRequest{Ops: map[uint64][]*ingest.Operation{shard: ops}}
+			continue
+		}
+		forThisShard.Ops[shard] = ops
+	}
+	eg, ctx := errgroup.WithContext(ctx)
+	for _, node := range snap.Nodes {
+		node := node
+		sharded := byNode[node.ID]
+		// Sometimes, there's nothing for a specific node.
+		if sharded == nil {
+			continue
+		}
+		if node.ID == api.NodeID() {
+			eg.Go(func() error {
+				return api.ingestNodeOperationsForFields(ctx, qcx, index, knownFields, sharded)
+			})
+		} else {
+			eg.Go(func() error {
+				return api.server.defaultClient.IngestNodeOperations(ctx, &node.URI, indexName, sharded)
+			})
+		}
 	}
 	return eg.Wait()
 }
@@ -3090,6 +3146,7 @@ const (
 	apiIDReset
 	apiPartitionNodes
 	apiIngestOperations
+	apiIngestNodeOperations
 	apiMutexCheck
 )
 
@@ -3159,5 +3216,6 @@ var methodsNormal = map[apiMethod]struct{}{
 	apiIDReset:              {},
 	apiPartitionNodes:       {},
 	apiIngestOperations:     {},
+	apiIngestNodeOperations: {},
 	apiMutexCheck:           {},
 }
diff --git a/client.go b/client.go
index 62f16e240..8726c440a 100644
--- a/client.go
+++ b/client.go
@@ -19,6 +19,7 @@ import (
 	"io"
 	"time"
 
+	"github.com/molecula/featurebase/v2/ingest"
 	pnet "github.com/molecula/featurebase/v2/net"
 	"github.com/molecula/featurebase/v2/topology"
 )
@@ -81,6 +82,7 @@ type InternalClient interface {
 	ImportRoaring(ctx context.Context, uri *pnet.URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error
 	ShardReader(ctx context.Context, index string, shard uint64) (io.ReadCloser, error)
 	MutexCheck(ctx context.Context, uri *pnet.URI, index string, field string, details bool, limit int) (map[uint64]map[uint64][]uint64, error)
+	IngestNodeOperations(ctx context.Context, uri *pnet.URI, indexName string, ireq *ingest.ShardedRequest) error
 
 	IDAllocDataReader(ctx context.Context) (io.ReadCloser, error)
 	IndexTranslateDataReader(ctx context.Context, index string, partitionID int) (io.ReadCloser, error)
@@ -218,6 +220,10 @@ func (n nopInternalClient) MutexCheck(ctx context.Context, uri *pnet.URI, index,
 	return nil, nil
 }
 
+func (n nopInternalClient) IngestNodeOperations(ctx context.Context, uri *pnet.URI, indexName string, ireq *ingest.ShardedRequest) error {
+	return nil
+}
+
 func (n nopInternalClient) ShardReader(ctx context.Context, index string, shard uint64) (io.ReadCloser, error) {
 	return nil, nil
 }
diff --git a/cluster.go b/cluster.go
index 6419f3681..29c4c2164 100644
--- a/cluster.go
+++ b/cluster.go
@@ -24,6 +24,7 @@ import (
 	"time"
 
 	"github.com/molecula/featurebase/v2/disco"
+	"github.com/molecula/featurebase/v2/ingest"
 	"github.com/molecula/featurebase/v2/logger"
 	"github.com/molecula/featurebase/v2/roaring"
 	"github.com/molecula/featurebase/v2/topology"
@@ -1566,6 +1567,39 @@ func (c *cluster) translateIndexKeys(ctx context.Context, indexName string, keys
 	return ids, nil
 }
 
+// This implements ingest's key translator interface on a cluster/index
+// pair.
+type clusterKeyTranslator struct {
+	ctx       context.Context // we're created within a request context and need to pass that to cluster ops
+	c         *cluster
+	indexName string
+}
+
+var _ ingest.KeyTranslator = &clusterKeyTranslator{}
+
+func (i clusterKeyTranslator) TranslateKeys(keys ...string) (map[string]uint64, error) {
+	return i.c.createIndexKeys(i.ctx, i.indexName, keys...)
+}
+
+func (i clusterKeyTranslator) TranslateIDs(ids ...uint64) (map[uint64]string, error) {
+	keys, err := i.c.translateIndexIDs(i.ctx, i.indexName, ids)
+	if err != nil {
+		return nil, err
+	}
+	if len(keys) != len(ids) {
+		return nil, fmt.Errorf("translating %d id(s), got %d key(s)", len(ids), len(keys))
+	}
+	out := make(map[uint64]string, len(keys))
+	for i, id := range ids {
+		out[id] = keys[i]
+	}
+	return out, nil
+}
+
+func newIngestKeyTranslatorFromCluster(ctx context.Context, c *cluster, indexName string) *clusterKeyTranslator {
+	return &clusterKeyTranslator{ctx: ctx, c: c, indexName: indexName}
+}
+
 // TODO: remove this when it is no longer used
 func (c *cluster) translateIndexKeySet(ctx context.Context, indexName string, keySet map[string]struct{}, writable bool) (map[string]uint64, error) {
 	keys := make([]string, 0, len(keySet))
diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go
index 69a456904..d765a8aff 100644
--- a/encoding/proto/proto.go
+++ b/encoding/proto/proto.go
@@ -21,6 +21,7 @@ import (
 	"github.com/gogo/protobuf/proto"
 	"github.com/molecula/featurebase/v2"
 	"github.com/molecula/featurebase/v2/disco"
+	"github.com/molecula/featurebase/v2/ingest"
 	pnet "github.com/molecula/featurebase/v2/net"
 	"github.com/molecula/featurebase/v2/pb"
 	"github.com/molecula/featurebase/v2/pql"
@@ -324,7 +325,18 @@ func (s Serializer) Unmarshal(buf []byte, m pilosa.Message) error {
 		}
 		decodeResizeAbortMessage(msg, mt)
 		return nil
-
+	case *ingest.ShardedRequest:
+		msg := &pb.ShardedIngestRequest{}
+		err := proto.Unmarshal(buf, msg)
+		if err != nil {
+			return errors.Wrap(err, "unmarshalling ShardedRequest")
+		}
+		req, err := s.decodeShardedIngestRequest(msg)
+		if err != nil {
+			return err
+		}
+		*mt = *req
+		return nil
 	default:
 		panic(fmt.Sprintf("unhandled pilosa.Message of type %T: %#v", mt, m))
 	}
@@ -398,6 +410,8 @@ func (s Serializer) encodeToProto(m pilosa.Message) proto.Message {
 		return s.encodeResizeNodeMessage(mt)
 	case *pilosa.ResizeAbortMessage:
 		return s.encodeResizeAbortMessage(mt)
+	case *ingest.ShardedRequest:
+		return s.encodeShardedIngestRequest(mt)
 	}
 	return nil
 }
@@ -931,6 +945,48 @@ func (s Serializer) encodeTransactionStats(stats pilosa.TransactionStats) *pb.Tr
 	return &pb.TransactionStats{}
 }
 
+func (s Serializer) encodeShardedIngestRequest(req *ingest.ShardedRequest) *pb.ShardedIngestRequest {
+	if req == nil || len(req.Ops) == 0 {
+		return &pb.ShardedIngestRequest{}
+	}
+	out := &pb.ShardedIngestRequest{Ops: make(map[uint64]*pb.ShardIngestOperations, len(req.Ops))}
+	for shard, ops := range req.Ops {
+		out.Ops[shard] = s.encodeShardIngestOperations(ops)
+	}
+	return out
+}
+
+func (s Serializer) encodeShardIngestOperations(ops []*ingest.Operation) *pb.ShardIngestOperations {
+	out := &pb.ShardIngestOperations{}
+	if len(ops) == 0 {
+		return out
+	}
+	for _, op := range ops {
+		if op == nil {
+			continue
+		}
+		out.Ops = append(out.Ops, s.encodeShardIngestOperation(op))
+	}
+	return out
+}
+
+func (s Serializer) encodeShardIngestOperation(op *ingest.Operation) *pb.ShardIngestOperation {
+	out := &pb.ShardIngestOperation{
+		OpType:         op.OpType.String(),
+		ClearRecordIDs: op.ClearRecordIDs,
+		ClearFields:    op.ClearFields,
+		FieldOps:       make(map[string]*pb.FieldOperation, len(op.FieldOps)),
+	}
+	for k, v := range op.FieldOps {
+		out.FieldOps[k] = &pb.FieldOperation{
+			RecordIDs: v.RecordIDs,
+			Values:    v.Values,
+			Signed:    v.Signed,
+		}
+	}
+	return out
+}
+
 func (s Serializer) decodeResizeInstruction(ri *pb.ResizeInstruction, m *pilosa.ResizeInstruction) {
 	m.JobID = ri.JobID
 	m.Node = &topology.Node{}
@@ -1829,3 +1885,57 @@ func decodeResizeNodeMessage(pb *pb.ResizeNodeMessage, m *pilosa.ResizeNodeMessa
 func decodeResizeAbortMessage(pb *pb.ResizeAbortMessage, m *pilosa.ResizeAbortMessage) {
 
 }
+
+func (s Serializer) decodeShardedIngestRequest(req *pb.ShardedIngestRequest) (*ingest.ShardedRequest, error) {
+	if req == nil || len(req.Ops) == 0 {
+		return &ingest.ShardedRequest{}, nil
+	}
+	out := &ingest.ShardedRequest{Ops: make(map[uint64][]*ingest.Operation, len(req.Ops))}
+	for shard, ops := range req.Ops {
+		var err error
+		out.Ops[shard], err = s.decodeShardIngestOperations(ops)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return out, nil
+}
+
+func (s Serializer) decodeShardIngestOperations(ops *pb.ShardIngestOperations) ([]*ingest.Operation, error) {
+	out := []*ingest.Operation{}
+	if len(ops.Ops) == 0 {
+		return out, nil
+	}
+	for _, op := range ops.Ops {
+		if op == nil {
+			continue
+		}
+		decoded, err := s.decodeShardIngestOperation(op)
+		if err != nil {
+			return nil, err
+		}
+		out = append(out, decoded)
+	}
+	return out, nil
+}
+
+func (s Serializer) decodeShardIngestOperation(op *pb.ShardIngestOperation) (*ingest.Operation, error) {
+	opType, err := ingest.ParseOpType(op.OpType)
+	if err != nil {
+		return nil, err
+	}
+	out := &ingest.Operation{
+		OpType:         opType,
+		ClearRecordIDs: op.ClearRecordIDs,
+		ClearFields:    op.ClearFields,
+		FieldOps:       make(map[string]*ingest.FieldOperation, len(op.FieldOps)),
+	}
+	for k, v := range op.FieldOps {
+		out.FieldOps[k] = &ingest.FieldOperation{
+			RecordIDs: v.RecordIDs,
+			Values:    v.Values,
+			Signed:    v.Signed,
+		}
+	}
+	return out, nil
+}
diff --git a/http/client.go b/http/client.go
index fb079218f..1d41a114f 100644
--- a/http/client.go
+++ b/http/client.go
@@ -32,6 +32,7 @@ import (
 
 	pilosa "github.com/molecula/featurebase/v2"
 	"github.com/molecula/featurebase/v2/encoding/proto"
+	"github.com/molecula/featurebase/v2/ingest"
 	pnet "github.com/molecula/featurebase/v2/net"
 	"github.com/molecula/featurebase/v2/topology"
 	"github.com/molecula/featurebase/v2/tracing"
@@ -284,6 +285,38 @@ func (c *InternalClient) IngestOperations(ctx context.Context, uri *pnet.URI, in
 	return nil
 }
 
+// IngestNodeOperations uses the internal/protobuf ingest endpoint for ingest data
+func (c *InternalClient) IngestNodeOperations(ctx context.Context, uri *pnet.URI, indexName string, ireq *ingest.ShardedRequest) error {
+	if uri == nil {
+		uri = c.defaultURI
+	}
+	u := uri.Path(fmt.Sprintf("/internal/ingest/%s/node", indexName))
+
+	buf, err := c.serializer.Marshal(ireq)
+	if err != nil {
+		return errors.Wrap(err, "marshalling")
+	}
+	req, err := http.NewRequest("POST", u, bytes.NewReader(buf))
+	if err != nil {
+		return errors.Wrap(err, "creating request")
+	}
+
+	req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
+	req.Header.Set("Content-Type", "application/x-protobuf")
+	req.Header.Set("Accept", "application/x-protobuf")
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+
+	resp, err := c.executeRequest(req.WithContext(ctx))
+	if err != nil {
+		return errors.Wrap(err, "executing request")
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return errors.Errorf("unexpected status code: %s", resp.Status)
+	}
+	return nil
+}
+
 // MutexCheck uses the mutex-check endpoint to request mutex collision data
 // from a single node. It produces per-shard results, and does not translate
 // them.
diff --git a/http/handler.go b/http/handler.go
index ac5a4dd68..d2eef2e5e 100644
--- a/http/handler.go
+++ b/http/handler.go
@@ -42,6 +42,7 @@ import (
 	"github.com/gorilla/mux"
 	pilosa "github.com/molecula/featurebase/v2"
 	"github.com/molecula/featurebase/v2/encoding/proto"
+	"github.com/molecula/featurebase/v2/ingest"
 	"github.com/molecula/featurebase/v2/logger"
 	"github.com/molecula/featurebase/v2/pql"
 	"github.com/molecula/featurebase/v2/rbf"
@@ -432,7 +433,9 @@ func newRouter(handler *Handler) http.Handler {
 	router.HandleFunc("/internal/index/{index}/shards", handler.handleGetIndexAvailableShards).Methods("GET").Name("GetIndexAvailableShards")
 	router.HandleFunc("/internal/nodes", handler.handleGetNodes).Methods("GET").Name("GetNodes")
 	router.HandleFunc("/internal/shards/max", handler.handleGetShardsMax).Methods("GET").Name("GetShardsMax") // TODO: deprecate, but it's being used by the client
-	router.HandleFunc("/internal/ingest/{index}", handler.handleIngestData).Methods("POST").Name("PostIngestData")
+	router.HandleFunc("/internal/ingest/{index}", handler.handlePostIngestData).Methods("POST").Name("PostIngestData")
+	router.HandleFunc("/internal/ingest/{index}/node", handler.handlePostIngestNode).Methods("POST").Name("PostIngestNode")
+
 	router.HandleFunc("/internal/schema", handler.handleIngestSchema).Methods("POST").Name("PostIngestSchema")
 	router.HandleFunc("/internal/translate/index/{index}/keys/find", handler.handleFindIndexKeys).Methods("POST").Name("FindIndexKeys")
 	router.HandleFunc("/internal/translate/index/{index}/keys/create", handler.handleCreateIndexKeys).Methods("POST").Name("CreateIndexKeys")
@@ -1337,7 +1340,9 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) {
 	resp.write(w, err)
 }
 
-func (h *Handler) handleIngestData(w http.ResponseWriter, r *http.Request) {
+// handlePostIngestData handles JSON ingest data that may need key
+// translation, for the entire cluster.
+func (h *Handler) handlePostIngestData(w http.ResponseWriter, r *http.Request) {
 	if !validHeaderAcceptJSON(r.Header) {
 		http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
 		return
@@ -1351,6 +1356,14 @@ func (h *Handler) handleIngestData(w http.ResponseWriter, r *http.Request) {
 
 	qcx := h.api.Txf().NewQcx()
 	err := h.api.IngestOperations(r.Context(), qcx, indexName, r.Body)
+	if err == nil {
+		err = qcx.Finish()
+		if err != nil {
+			http.Error(w, fmt.Sprintf("ingesting: %v", err), http.StatusInternalServerError)
+		}
+	} else {
+		qcx.Abort()
+	}
 
 	resp := successResponse{h: h, Name: indexName}
 	resp.write(w, err)
@@ -2908,6 +2921,52 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request
 	}
 }
 
+// handlePostIngestNode is the internal endpoint taking already-translated
+// ingest operations, sorted by shard, for a single node.
+func (h *Handler) handlePostIngestNode(w http.ResponseWriter, r *http.Request) {
+	// Verify that request is only communicating over protobufs.
+	if error, code := validateProtobufHeader(r); error != "" {
+		http.Error(w, error, code)
+		return
+	}
+
+	ctx := r.Context()
+
+	// Read entire body.
+	span, _ := tracing.StartSpanFromContext(ctx, "ioutil.ReadAll-Body")
+	body, err := readBody(r)
+	span.LogKV("bodySize", len(body))
+	span.Finish()
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	req := &ingest.ShardedRequest{}
+	span, _ = tracing.StartSpanFromContext(ctx, "Unmarshal")
+	err = proto.DefaultSerializer.Unmarshal(body, req)
+	span.Finish()
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	urlVars := mux.Vars(r)
+	indexName := urlVars["index"]
+
+	qcx := h.api.Txf().NewQcx()
+	err = h.api.IngestNodeOperations(r.Context(), qcx, indexName, req)
+	if err == nil {
+		err = qcx.Finish()
+		if err != nil {
+			http.Error(w, fmt.Sprintf("ingesting: %v", err), http.StatusInternalServerError)
+		}
+	} else {
+		http.Error(w, fmt.Sprintf("ingesting: %v", err), http.StatusInternalServerError)
+		qcx.Abort()
+	}
+}
+
 func (h *Handler) handlePostTranslateKeys(w http.ResponseWriter, r *http.Request) {
 	// Verify that request is only communicating over protobufs.
 	if r.Header.Get("Content-Type") != "application/x-protobuf" {
diff --git a/ingest/codec.go b/ingest/codec.go
index c966e3f1c..91b63000f 100644
--- a/ingest/codec.go
+++ b/ingest/codec.go
@@ -15,10 +15,13 @@
 package ingest
 
 import (
+	"bytes"
+	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"math"
+	"sort"
 	"strconv"
 	"time"
 
@@ -37,15 +40,22 @@ import (
 // decimal	yes	yes	no
 // timestamp	yes	yes	no
 
-type KeyLookupFunc func(...string) (map[string]uint64, error)
+// KeyTranslator is a thing that can translate strings to IDs, and also
+// IDs back to strings. The ID->string conversion is used only to render
+// a request back to JSON, which in turn is only used in testing. The
+// functions optionally take an existing map which they then augment.
+type KeyTranslator interface {
+	TranslateKeys(keys ...string) (map[string]uint64, error)
+	TranslateIDs(ids ...uint64) (map[uint64]string, error)
+}
 
 // Codec is a single-use parser which decodes data into columnar vectors.
 type Codec interface {
-	AddSetField(name string, lookup KeyLookupFunc) error
-	AddTimeQuantumField(name string, lookup KeyLookupFunc) error
-	AddMutexField(name string, lookup KeyLookupFunc) error
+	AddSetField(name string, keys KeyTranslator) error
+	AddTimeQuantumField(name string, keys KeyTranslator) error
+	AddMutexField(name string, keys KeyTranslator) error
 	AddBoolField(name string) error
-	AddIntField(name string, lookup KeyLookupFunc) error
+	AddIntField(name string, keys KeyTranslator) error
 	AddDecimalField(name string, scale int64) error
 	AddTimestampField(name string, scale time.Duration, epoch int64) error
 
@@ -56,138 +66,299 @@ type Codec interface {
 
 type jsonDecFn func(recID uint64, typ jsonparser.ValueType, data []byte) error
 
-// applyTranslationFn is a function which applies key lookups to the values
-// of an operation, meaning it needs to know whether it's applying them
-// to the signed or unsigned values.
-type applyTranslationFn func(*FieldOperation, []uint64) error
+// jsonEncFn encodes a value, or range of values, according to a given
+// fieldCodec's translation rules. key values, if present, will be used to
+// replace whichever values could have been provided as keys. so for instance,
+// with an int field which is keyed, values are actually of type int64, but
+// if strings are present, those are used. for a time quantum field, the time
+// is set from the signed field, and the value is replaced by the key if
+// keys are provided.
+type jsonEncFn func(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error
 
-type jsonFieldCodec struct {
+// fieldCodec represents something that can encode and decode a
+// particular field. Its methods are not reentrant, it uses internal
+// buffers.
+type fieldCodec struct {
 	valueKeys *StringTable
 	currentOp *FieldOperation
 	decode    jsonDecFn
-	translate applyTranslationFn
+	encode    jsonEncFn
 	// For timestamp: scale-in-nanoseconds; for instance, if scaleUnit is
 	// 1,000,000,000, we are storing numbers-of-seconds since the Unix epoch.
 	// The actual value recorded in BSI will be offset by the field's
 	// epoch, but we don't need to know that.
 	// For decimal: Decimal digits of precision. So for instance, with
-	// scaleUnit 2, "1" is stored as 100 and "1.2" is stored as 120.
+	// scale 2, scaleUnit is 100, "1" is stored as 100 and "1.2" is stored as
+	// 120.
 	scaleUnit int64
+	scale     int64
 	epoch     int64    // used only by Timestamp fields
 	scratch   []uint64 // reusable scratch space for sets of values
-	lookup    KeyLookupFunc
+	keys      KeyTranslator
+	buf       []byte // scratch space for format operations.
 	fieldType FieldType
 }
 
 // JSONCodec is a Codec which accepts a JSON map of record keys/ids to updated value maps.
 type JSONCodec struct {
-	recKeys   *StringTable
-	fields    map[string]*jsonFieldCodec
-	keyLookup KeyLookupFunc
-	currentOp *Operation
+	fieldTypes map[string]FieldType
+	recKeys    *StringTable
+	fields     map[string]*fieldCodec
+	keys       KeyTranslator
+	currentOp  *Operation
+}
+
+// jsonBuffer is a bytes.Buffer which has an associated json.Encoder which
+// can be used to write to its buffer. Both types are just embedded because
+// they have non-overlapping APIs. Please don't look at my horrible face.
+//
+// This has some internal objects it can use to stash things
+// so that it can pass &foo to enc.Encode() without needing to heap-allocate
+// something for the interface conversion, and a buffer it can use for
+// converting numbers or times.
+//
+// It also has an internal error buffer. In fact, in many cases, errors
+// are so far as I can tell absolutely impossible -- bytes.Buffer specifically
+// promises never to yield an error, and nothing seems to hint that
+// enc.Encode can error on integers, strings, booleans, or arrays. So we
+// have dozens of error checks that we can't cause to happen even with
+// malformed inputs... so we eat those errors, don't require them to be
+// checked externally, and return them when done if anyone checks.
+type jsonBuffer struct {
+	*bytes.Buffer
+	enc     *json.Encoder
+	buf     [48]byte // scratch space for using strconv.AppendInt, etc
+	uintbuf []uint64 // dummy buffer so that we don't have to alloc to print []uint64
+	strbuf  []string // dummy buffer so that we don't have to alloc to print []string
+	str     string   // and again, "so we don't have to alloc a copy"
+	err     error
+}
+
+// Encode removes the stray newlines added by json.Encoder.
+func (j *jsonBuffer) Encode(v interface{}) {
+	err := j.enc.Encode(v)
+	if err == nil {
+		j.Truncate(j.Len() - 1)
+	} else {
+		j.err = err
+	}
+}
+
+// EncodeInt uses AppendInt into a static buffer to reduce allocs.
+func (j *jsonBuffer) EncodeInt(i int64) {
+	rep := strconv.AppendInt(j.buf[:0], i, 10)
+	_, _ = j.Write(rep)
+}
+
+// EncodeUint uses AppendUint into a static buffer to reduce allocs.
+func (j *jsonBuffer) EncodeUint(u uint64) {
+	rep := strconv.AppendUint(j.buf[:0], u, 10)
+	_, _ = j.Write(rep)
+}
+
+// EncodeUints uses a static copy of a []uint64 -- the slice, not its
+// contents --  in already-allocated memory so the interface conversion
+// doesn't have to do that.
+func (j *jsonBuffer) EncodeUints(u []uint64) {
+	j.uintbuf = u
+	j.Encode(&j.uintbuf)
+	j.strbuf = nil
+}
+
+// EncodeStrings uses a static copy of a []string -- the slice, not its
+// contents --  in already-allocated memory so the interface conversion
+// doesn't have to do that.
+func (j *jsonBuffer) EncodeStrings(s []string) {
+	j.strbuf = s
+	j.Encode(&j.strbuf)
+	j.strbuf = nil
+}
+
+// EncodeQuotedUint uses AppendUint into a static buffer to reduce allocs,
+// while also surrounding the value with quotes.
+func (j *jsonBuffer) EncodeQuotedUint(u uint64) {
+	j.buf[0] = '"'
+	rep := strconv.AppendUint(j.buf[1:1], u, 10)
+	j.buf[len(rep)+1] = '"'
+	_, _ = j.Write(j.buf[:len(rep)+2])
+}
+
+// EncodeString appends the JSON encoding of a string. This exists
+// because otherwise runtime allocates a heap-allocated copy of the
+// string to live inside an interface{} for the duration of a function
+// call...
+func (j *jsonBuffer) EncodeString(s string) {
+	j.str = s
+	j.Encode(&j.str)
+	j.str = ""
+}
+
+// EncodeTime exists because time's MarshalJSON allocates a new
+// buffer every time it gets called, resulting in a full 13% of
+// all the allocations produced in a test run, plus another 13%
+// or so of them which were for the copies of the time objects
+// made to stuff them into an interface{}. Eww.
+func (j *jsonBuffer) EncodeTime(t time.Time) {
+	j.buf[0] = '"'
+	rep := t.AppendFormat(j.buf[1:1], time.RFC3339Nano)
+	j.buf[len(rep)+1] = '"'
+	_, _ = j.Write(j.buf[:len(rep)+2])
+}
+
+// EncodeBool writes a literal representation directly to reduce allocs.
+func (j *jsonBuffer) EncodeBool(b bool) {
+	if b {
+		_, _ = j.WriteString("true")
+	} else {
+		_, _ = j.WriteString("false")
+	}
+}
+
+func (j *jsonBuffer) Err() error {
+	return j.err
+}
+
+func newJSONBuffer(data []byte) *jsonBuffer {
+	e := &jsonBuffer{Buffer: bytes.NewBuffer(data)}
+	e.enc = json.NewEncoder(e.Buffer)
+	e.enc.SetEscapeHTML(false) // we are not doing HTML, just JSON
+	return e
 }
 
 var _ Codec = &JSONCodec{}
 
-func NewJSONCodec(lookup KeyLookupFunc) (*JSONCodec, error) {
-	j := &JSONCodec{fields: map[string]*jsonFieldCodec{}}
-	if lookup != nil {
+func NewJSONCodec(keys KeyTranslator) (*JSONCodec, error) {
+	j := &JSONCodec{
+		fields:     map[string]*fieldCodec{},
+		fieldTypes: map[string]FieldType{},
+	}
+	if keys != nil {
 		j.recKeys = NewStringTable()
-		j.keyLookup = lookup
+		j.keys = keys
 	}
 	return j, nil
 }
 
-func (codec *JSONCodec) AddTimeQuantumField(name string, lookup KeyLookupFunc) error {
-	fieldCodec := &jsonFieldCodec{}
+func (codec *JSONCodec) AddTimeQuantumField(name string, keys KeyTranslator) error {
+	fieldCodec := &fieldCodec{
+		fieldType: FieldTypeTimeQuantum,
+		keys:      keys,
+	}
 	fieldCodec.decode = fieldCodec.DecodeTimeQuantumValue
-	if lookup != nil {
-		fieldCodec.translate = (*FieldOperation).TranslateUnsigned
-	}
-	return codec.addField(name, FieldTypeTimeQuantum, fieldCodec, lookup)
+	fieldCodec.encode = fieldCodec.EncodeTimeQuantumValue
+	return codec.addField(name, fieldCodec)
 }
 
-func (codec *JSONCodec) AddSetField(name string, lookup KeyLookupFunc) error {
-	fieldCodec := &jsonFieldCodec{}
+func (codec *JSONCodec) AddSetField(name string, keys KeyTranslator) error {
+	fieldCodec := &fieldCodec{
+		fieldType: FieldTypeSet,
+		keys:      keys,
+	}
 	fieldCodec.decode = fieldCodec.DecodeSetValue
-	if lookup != nil {
-		fieldCodec.translate = (*FieldOperation).TranslateUnsigned
-	}
-	return codec.addField(name, FieldTypeSet, fieldCodec, lookup)
+	fieldCodec.encode = fieldCodec.EncodeSetValue
+	return codec.addField(name, fieldCodec)
 }
 
-func (codec *JSONCodec) AddIntField(name string, lookup KeyLookupFunc) error {
-	fieldCodec := &jsonFieldCodec{}
+func (codec *JSONCodec) AddIntField(name string, keys KeyTranslator) error {
+	fieldCodec := &fieldCodec{
+		fieldType: FieldTypeInt,
+		keys:      keys,
+	}
 	fieldCodec.decode = fieldCodec.DecodeIntValue
-	if lookup != nil {
-		fieldCodec.translate = (*FieldOperation).TranslateSigned
-	}
-	return codec.addField(name, FieldTypeInt, fieldCodec, lookup)
+	fieldCodec.encode = fieldCodec.EncodeIntValue
+	return codec.addField(name, fieldCodec)
 }
 
-func (codec *JSONCodec) AddMutexField(name string, lookup KeyLookupFunc) error {
-	fieldCodec := &jsonFieldCodec{}
-	fieldCodec.decode = fieldCodec.DecodeMutexValue
-	if lookup != nil {
-		fieldCodec.translate = (*FieldOperation).TranslateUnsigned
+func (codec *JSONCodec) AddMutexField(name string, keys KeyTranslator) error {
+	fieldCodec := &fieldCodec{
+		fieldType: FieldTypeMutex,
+		keys:      keys,
 	}
-	return codec.addField(name, FieldTypeMutex, fieldCodec, lookup)
+	fieldCodec.decode = fieldCodec.DecodeMutexValue
+	fieldCodec.encode = fieldCodec.EncodeMutexValue
+	return codec.addField(name, fieldCodec)
 }
 
 func (codec *JSONCodec) AddBoolField(name string) error {
-	fieldCodec := &jsonFieldCodec{}
+	fieldCodec := &fieldCodec{
+		fieldType: FieldTypeBool,
+	}
 	fieldCodec.decode = fieldCodec.DecodeBoolValue
-	return codec.addField(name, FieldTypeBool, fieldCodec, nil)
+	fieldCodec.encode = fieldCodec.EncodeBoolValue
+	return codec.addField(name, fieldCodec)
 }
 
 // TimestampField is used to store seconds since unix epoch. The numeric values
 // stored are adjusted based on the given time.Duration; for instance, if the
 // time scale is time.Second, then the second after the epoch is stored as 1,
-// if it's time.Millisecond, then it's stored as 1000, etcetera.
+// if it's time.Millisecond, then it's stored as 1000, etcetera. The epoch
+// passed to this function should be the offset from the Unix epoch to the
+// desired epoch, in the same scale. (So if the scale is milliseconds,
+// it should be the Unix timestamp in seconds, times 1000.)
 func (codec *JSONCodec) AddTimestampField(name string, timeScale time.Duration, epoch int64) error {
-	fieldCodec := &jsonFieldCodec{scaleUnit: int64(timeScale), epoch: epoch}
+	fieldCodec := &fieldCodec{
+		fieldType: FieldTypeTimeStamp,
+		scaleUnit: int64(timeScale),
+		epoch:     epoch,
+	}
 	fieldCodec.decode = fieldCodec.DecodeTimeValue
-	return codec.addField(name, FieldTypeTimeStamp, fieldCodec, nil)
+	fieldCodec.encode = fieldCodec.EncodeTimeValue
+	return codec.addField(name, fieldCodec)
 }
 
 // AddDecimalField adds a decimal field, which is stored as integer values
 // with a scale offset, but parsed as floating point values. For instance,
 // with decimalScale=2, `0.01` would store the value 1.
 func (codec *JSONCodec) AddDecimalField(name string, decimalScale int64) error {
-	fieldCodec := &jsonFieldCodec{scaleUnit: int64(math.Pow10(int(decimalScale)))}
+	fieldCodec := &fieldCodec{
+		fieldType: FieldTypeDecimal,
+		scale:     decimalScale,
+		scaleUnit: int64(math.Pow(10, float64(decimalScale))),
+	}
 	fieldCodec.decode = fieldCodec.DecodeDecimalValue
-	return codec.addField(name, FieldTypeDecimal, fieldCodec, nil)
+	fieldCodec.encode = fieldCodec.EncodeDecimalValue
+	return codec.addField(name, fieldCodec)
 }
 
-func (codec *JSONCodec) addField(name string, fieldType FieldType, fieldCodec *jsonFieldCodec, lookup KeyLookupFunc) error {
+func (codec *JSONCodec) addField(name string, fieldCodec *fieldCodec) error {
 	if _, ok := codec.fields[name]; ok {
 		return fmt.Errorf("duplicate field %q", name)
 	}
-	if lookup != nil {
+	if fieldCodec.keys != nil {
 		fieldCodec.valueKeys = NewStringTable()
-		fieldCodec.lookup = lookup
 	}
-	fieldCodec.fieldType = fieldType
 	codec.fields[name] = fieldCodec
+	codec.fieldTypes[name] = fieldCodec.fieldType
 	return nil
 }
 
 // decodeSetOrValue decodes a value which might be either an array of values or a
 // single value, where values might be string keys or bare numbers, calling cb for
 // each value it finds.
-func (j *jsonFieldCodec) decodeSetOrValue(dataType jsonparser.ValueType, data []byte, cb func(uint64) error) (err error) {
+func (j *fieldCodec) decodeSetOrValue(dataType jsonparser.ValueType, data []byte, cb func(uint64) error) (err error) {
 	switch dataType {
 	case jsonparser.Array:
 		_, arrayErr := jsonparser.ArrayEach(data, func(value []byte, dataType jsonparser.ValueType, offset int, unused error) {
-			id, idErr := j.valueKeys.ID(value)
-			if idErr != nil {
-				err = idErr
-				return
-			}
-			// stash an error if we got one
-			valueErr := cb(id)
-			if valueErr != nil {
-				err = valueErr
+			var id uint64
+			switch dataType {
+			case jsonparser.String:
+				id, err = j.valueKeys.ID(value)
+				if err != nil {
+					return
+				}
+				err = cb(id)
+			case jsonparser.Number:
+				if j.valueKeys != nil {
+					err = errors.New("expecting key, got numeric value")
+					return
+				}
+				id, err = strconv.ParseUint(pretendByteIsString(value), 10, 64)
+				if err != nil {
+					return
+				}
+				err = cb(id)
+			default:
+				err = fmt.Errorf("expecting value or array, got %v", dataType)
 			}
 		})
 		if err != nil {
@@ -212,23 +383,32 @@ func (j *jsonFieldCodec) decodeSetOrValue(dataType jsonparser.ValueType, data []
 		}
 		return cb(id)
 	default:
-		return fmt.Errorf("expecting array, got %v", dataType)
+		return fmt.Errorf("expecting value or array, got %v", dataType)
 	}
 	return err
 }
 
 // DecodeSetValue decodes a set of unsigned values from the provided data into the
 // associated currentOp.
-func (j *jsonFieldCodec) DecodeSetValue(recID uint64, dataType jsonparser.ValueType, data []byte) (err error) {
+func (j *fieldCodec) DecodeSetValue(recID uint64, dataType jsonparser.ValueType, data []byte) (err error) {
 	return j.decodeSetOrValue(dataType, data, func(id uint64) error {
 		j.currentOp.AddPair(recID, id)
 		return nil
 	})
 }
 
+func (j *fieldCodec) EncodeSetValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error {
+	if len(keys) == 1 || len(values) == 1 {
+		// simplify: just hand this off to a single-value case
+		return j.EncodeMutexValue(dst, values, signed, keys)
+	}
+	appendKeysJSON(dst, values, keys)
+	return nil
+}
+
 // DecodeIntValue decodes a single signed value from the provided data
 // into the associated currentOp.
-func (j *jsonFieldCodec) DecodeIntValue(recID uint64, dataType jsonparser.ValueType, data []byte) error {
+func (j *fieldCodec) DecodeIntValue(recID uint64, dataType jsonparser.ValueType, data []byte) error {
 	switch dataType {
 	case jsonparser.String:
 		value, err := j.valueKeys.IntID(data)
@@ -255,25 +435,62 @@ func (j *jsonFieldCodec) DecodeIntValue(recID uint64, dataType jsonparser.ValueT
 	return nil
 }
 
+func (j *fieldCodec) EncodeIntValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error {
+	if len(keys) == 0 {
+		if len(signed) == 0 {
+			return errors.New("encodeIntValue: need a value")
+		}
+		dst.EncodeInt(signed[0])
+		return nil
+	}
+	dst.EncodeString(keys[0])
+	return nil
+}
+
 // DecodeMutexValue decodes a single unsigned value from the provided data
 // into the associated currentOp.
-func (j *jsonFieldCodec) DecodeMutexValue(recID uint64, dataType jsonparser.ValueType, data []byte) error {
+func (j *fieldCodec) DecodeMutexValue(recID uint64, dataType jsonparser.ValueType, data []byte) error {
 	switch dataType {
-	case jsonparser.Number, jsonparser.String:
-		id, err := j.valueKeys.ID(data)
+	case jsonparser.String:
+		value, err := j.valueKeys.ID(data)
 		if err != nil {
 			return err
 		}
-		j.currentOp.AddPair(recID, id)
+		j.currentOp.AddPair(recID, value)
+	case jsonparser.Number:
+		if j.valueKeys != nil {
+			return errors.New("expecting string key, got numeric value")
+		}
+		value, err := strconv.ParseUint(pretendByteIsString(data), 10, 64)
+		if err != nil {
+			return err
+		}
+		j.currentOp.AddPair(recID, value)
 	default:
-		return fmt.Errorf("expecting integer value, got %v", dataType)
+		if j.valueKeys != nil {
+			return fmt.Errorf("expecting string key, got %v", dataType)
+		} else {
+			return fmt.Errorf("expecting integer value, got %v", dataType)
+		}
 	}
 	return nil
 }
 
+func (j *fieldCodec) EncodeMutexValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error {
+	if len(keys) == 0 {
+		if len(values) == 0 {
+			return errors.New("encodeMutexValue: need a value")
+		}
+		dst.EncodeUint(values[0])
+		return nil
+	}
+	dst.EncodeString(keys[0])
+	return nil
+}
+
 // DecodeBoolValue decodes a single true/false value from the provided data
 // into the associated currentOp.
-func (j *jsonFieldCodec) DecodeBoolValue(recID uint64, typ jsonparser.ValueType, data []byte) error {
+func (j *fieldCodec) DecodeBoolValue(recID uint64, typ jsonparser.ValueType, data []byte) error {
 	value := uint64(0)
 	switch typ {
 	case jsonparser.String:
@@ -303,9 +520,21 @@ func (j *jsonFieldCodec) DecodeBoolValue(recID uint64, typ jsonparser.ValueType,
 	return nil
 }
 
+func (j *fieldCodec) EncodeBoolValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error {
+	if len(values) == 0 {
+		return errors.New("encoding boolean value, but none provided")
+	}
+	var x bool
+	if values[0] != 0 {
+		x = true
+	}
+	dst.EncodeBool(x)
+	return nil
+}
+
 // DecodeTimeQuantumValue decodes a timestamp, and a set of bits from the
 // provided data into the associated currentOp.
-func (j *jsonFieldCodec) DecodeTimeQuantumValue(recID uint64, typ jsonparser.ValueType, data []byte) error {
+func (j *fieldCodec) DecodeTimeQuantumValue(recID uint64, typ jsonparser.ValueType, data []byte) error {
 	j.scratch = j.scratch[:0]
 	stamp := time.Unix(0, 0).UTC()
 	err := jsonparser.ObjectEach(data, func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) (err error) {
@@ -346,8 +575,27 @@ func (j *jsonFieldCodec) DecodeTimeQuantumValue(recID uint64, typ jsonparser.Val
 	return nil
 }
 
+// Our format does not allow a record to have multiple values set with
+// different timestamps at the same time. We use the first timestamp for
+// the whole set.
+func (j *fieldCodec) EncodeTimeQuantumValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error {
+	if len(values) == 0 {
+		return errors.New("encoding time quantum value: no value provided")
+	}
+	dst.WriteString(`{"values":`)
+	appendKeysJSON(dst, values, keys)
+	// a zero timestamp is idiomatic for no-time-provided, and i sort of
+	// hate that, but here we are.
+	if len(signed) > 0 && signed[0] != 0 {
+		_, _ = dst.WriteString(`,"time":`)
+		dst.EncodeTime(time.Unix(0, signed[0]).UTC())
+	}
+	_, _ = dst.WriteString(`}`)
+	return nil
+}
+
 // DecodeTimeValue will eventually work but right now it doesn't actually.
-func (j *jsonFieldCodec) DecodeTimeValue(recID uint64, dataType jsonparser.ValueType, data []byte) (err error) {
+func (j *fieldCodec) DecodeTimeValue(recID uint64, dataType jsonparser.ValueType, data []byte) (err error) {
 	var stamp time.Time
 	switch dataType {
 	case jsonparser.String:
@@ -364,12 +612,22 @@ func (j *jsonFieldCodec) DecodeTimeValue(recID uint64, dataType jsonparser.Value
 			return fmt.Errorf("parsing numeric timestamp: %w", err)
 		}
 		j.currentOp.AddSignedPair(recID, i64)
+	default:
+		return fmt.Errorf("expecting time, got %s", dataType)
 	}
 	return nil
 }
 
+func (j *fieldCodec) EncodeTimeValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error {
+	if len(signed) == 0 {
+		return errors.New("encoding time value: no value provided")
+	}
+	dst.EncodeTime(time.Unix(0, (signed[0]+j.epoch)*j.scaleUnit).UTC())
+	return nil
+}
+
 // DecodeDecimalValue will eventually work but right now it doesn't actually.
-func (j *jsonFieldCodec) DecodeDecimalValue(recID uint64, dataType jsonparser.ValueType, data []byte) error {
+func (j *fieldCodec) DecodeDecimalValue(recID uint64, dataType jsonparser.ValueType, data []byte) error {
 	switch dataType {
 	case jsonparser.String, jsonparser.Number:
 		value, err := jsonparser.GetFloat(data)
@@ -384,6 +642,28 @@ func (j *jsonFieldCodec) DecodeDecimalValue(recID uint64, dataType jsonparser.Va
 	return nil
 }
 
+func (j *fieldCodec) EncodeDecimalValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error {
+	j.buf = strconv.AppendInt(j.buf[:0], signed[0], 10)
+	scale := int(j.scale)
+	if len(j.buf) > scale {
+		j.buf = append(j.buf, '.')
+		// shove the last scaleUnit values over
+		copy(j.buf[len(j.buf)-scale:], j.buf[len(j.buf)-scale-1:])
+		j.buf[len(j.buf)-scale-1] = '.'
+	}
+	_, _ = dst.Write(j.buf)
+	return nil
+}
+
+// FieldTypes gives a mapping of fields to their basic types used by this
+// codec.
+func (codec *JSONCodec) FieldTypes() map[string]FieldType {
+	return codec.fieldTypes
+}
+
+// ParseKeyedRecords parses the records it finds. Note that, when you're doing
+// a Write op, this will update ClearRecordIDs automatically as it goes,
+// even though record_ids isn't specified in the JSON for that case.
 func (codec *JSONCodec) ParseKeyedRecords(data []byte) (err error) {
 	seen := make(map[uint64]struct{})
 	return jsonparser.ObjectEach(data, func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) error {
@@ -412,8 +692,8 @@ func (codec *JSONCodec) ParseKeyedRecords(data []byte) (err error) {
 	})
 }
 
-func (codec *JSONCodec) ParseOperation(data []byte) (op *Operation, err error) {
-	op = &Operation{FieldOps: make(map[string]*FieldOperation, len(codec.fields))}
+func (codec *JSONCodec) ParseOperation(data []byte, seq int) (op *Operation, err error) {
+	op = &Operation{FieldOps: make(map[string]*FieldOperation, len(codec.fields)), Seq: seq}
 	codec.currentOp = op
 	err = jsonparser.ObjectEach(data, func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) error {
 		switch string(key) {
@@ -491,10 +771,12 @@ func (codec *JSONCodec) Parse(r io.Reader) (req *Request, err error) {
 func (codec *JSONCodec) ParseBytes(data []byte) (req *Request, err error) {
 	var ops []*Operation
 	var lastErr error
+	var seq int
 	_, err = jsonparser.ArrayEach(data, func(value []byte, dataType jsonparser.ValueType, offset int, err error) {
 		switch dataType {
 		case jsonparser.Object:
-			op, err := codec.ParseOperation(value)
+			op, err := codec.ParseOperation(value, seq)
+			seq++
 			if err != nil {
 				lastErr = fmt.Errorf("parsing operation: %v", err)
 				return
@@ -512,34 +794,39 @@ func (codec *JSONCodec) ParseBytes(data []byte) (req *Request, err error) {
 	}
 	// and now, key translation!
 	var keyMap []uint64
-	if codec.keyLookup != nil {
-		keyMap, err = MapForStringTable(codec.recKeys, codec.keyLookup)
+	if codec.keys != nil {
+		keyMap, err = codec.recKeys.MakeIDMap(codec.keys)
 		if err != nil {
 			return nil, fmt.Errorf("trying to find record key mapping: %w", err)
 		}
 	}
-	req = &Request{FieldTypes: make(map[string]FieldType, len(codec.fields))}
+	req = &Request{}
 	valueMaps := map[string]func(*FieldOperation) error{}
 	for name, fieldCodec := range codec.fields {
 		// make closure survive iteration
 		fieldCodec := fieldCodec
-		if fieldCodec.lookup != nil {
-			fieldMap, err := MapForStringTable(fieldCodec.valueKeys, fieldCodec.lookup)
+		if fieldCodec.keys != nil {
+			fieldMap, err := fieldCodec.valueKeys.MakeIDMap(fieldCodec.keys)
 			if err != nil {
 				return nil, fmt.Errorf("trying to find value mapping for %q: %w", name, err)
 			}
-			valueMaps[name] = func(fo *FieldOperation) error {
-				return fieldCodec.translate(fo, fieldMap)
+			if fieldCodec.fieldType == FieldTypeInt {
+				valueMaps[name] = func(fo *FieldOperation) error {
+					return translateSigned(fieldMap, fo.Signed)
+				}
+			} else {
+				valueMaps[name] = func(fo *FieldOperation) error {
+					return translateUnsigned(fieldMap, fo.Values)
+				}
 			}
 		}
-		req.FieldTypes[name] = fieldCodec.fieldType
 	}
 	for _, op := range ops {
 		// For Clear and Write, we need to translate/sort our record ID
 		// list.
 		if op.OpType == OpClear || op.OpType == OpWrite || op.OpType == OpDelete {
 			if keyMap != nil {
-				if err = translateUnsignedSlice(op.ClearRecordIDs, keyMap); err != nil {
+				if err = translateUnsigned(keyMap, op.ClearRecordIDs); err != nil {
 					return nil, fmt.Errorf("mapping record keys for clear op: %w", err)
 				}
 			}
@@ -554,7 +841,7 @@ func (codec *JSONCodec) ParseBytes(data []byte) (req *Request, err error) {
 				continue
 			}
 			if keyMap != nil {
-				if err = fieldOp.TranslateKeys(keyMap); err != nil {
+				if err = translateUnsigned(keyMap, fieldOp.RecordIDs); err != nil {
 					return nil, fmt.Errorf("mapping record keys for op on %q: %w", field, err)
 				}
 			}
@@ -574,6 +861,283 @@ func (codec *JSONCodec) ParseBytes(data []byte) (req *Request, err error) {
 	return req, nil
 }
 
+// AppendBytes appends bytes which we would expect to produce the same
+// request, using this codec. It does not try to recreate FieldTypes, which
+// would be handled by the codec anyway. It's written as an Append so you
+// can reuse a buffer.
+func (codec *JSONCodec) AppendBytes(req *Request, data []byte) (out []byte, err error) {
+	if req == nil || len(req.Ops) == 0 {
+		return append(data, "[]"...), nil
+	}
+	dst := newJSONBuffer(data)
+	// we ignore the FieldTypes part of the Request, which is just there to
+	// let the request's ByShard use the correct sorting routines, which is
+	// itself sort of awful. The codec will insert it again when parsing the
+	// bytes.
+	_, _ = dst.WriteString(`,`)
+	for _, op := range req.Ops {
+		// EncodeJSON needs to have access to this codec's field data
+		// and key translators.
+		err = op.EncodeJSON(dst, codec)
+		if err != nil {
+			return nil, err
+		}
+		_, _ = dst.WriteString(`,`)
+	}
+	// delete the last comma
+	dst.Truncate(dst.Len() - 1)
+	_, _ = dst.WriteString(`]`)
+	// there's very few ways an error could occur, possibly none, but
+	// just in case lots of internal operations stashed an error if one
+	// happened, so we'll return anything they came up with.
+	return dst.Bytes(), dst.Err()
+}
+
+// appendIDsJSON appends the provided IDs to a JSON stream as a bracketed
+// list of quoted strings if there's a key translator, or integer values
+// if the KeyTranslator is nil, comma-separated. It can error because a
+// translator can error.
+func (codec *JSONCodec) appendIDsJSON(dst *jsonBuffer, values []uint64, keys KeyTranslator) error {
+	return appendIDsJSON(dst, values, keys)
+}
+
+func appendKeysJSON(dst *jsonBuffer, values []uint64, keys []string) {
+	if len(values) == 0 && len(keys) == 0 {
+		_, _ = dst.WriteString("[]")
+		return
+	}
+	if len(keys) != 0 {
+		dst.EncodeStrings(keys)
+	} else {
+		dst.EncodeUints(values)
+	}
+}
+
+func appendIDsJSON(dst *jsonBuffer, values []uint64, keys KeyTranslator) error {
+	if len(values) == 0 {
+		_, _ = dst.WriteString("[]")
+		return nil
+	}
+
+	if keys == nil {
+		dst.EncodeUints(values)
+		return nil
+	}
+	_, _ = dst.WriteString(`[`)
+	translated, err := keys.TranslateIDs(values...)
+	if err != nil {
+		return err
+	}
+	for _, v := range values {
+		dst.EncodeString(translated[v])
+		_, _ = dst.WriteString(`,`)
+	}
+	dst.Truncate(dst.Len() - 1)
+	_, _ = dst.WriteString(`]`)
+	return nil
+}
+
+// RequestByShard makes up for the fact that we don't want to stash the
+// field type data in the request, but we need it to actually do the by-shard.
+func (codec *JSONCodec) RequestByShard(req *Request) (*ShardedRequest, error) {
+	return req.ByShard(codec.fieldTypes)
+}
+
+// EncodeJSON encodes an operation as JSON using a provided buffer.
+// It uses the provided codec where necessary to help with key
+// translation.
+func (o *Operation) EncodeJSON(dst *jsonBuffer, codec *JSONCodec) (err error) {
+	// We're not trying to guarantee that what we produce makes sense or is
+	// identical to what produced us, just to write our current state out.
+	if o == nil {
+		_, _ = dst.WriteString(`{}`)
+		return nil
+	}
+	_, _ = dst.WriteString(`{"action":`)
+	dst.EncodeString(o.OpType.String())
+	// it is intentional that o.Seq isn't encoded here; it makes no sense
+	// to allow an op to specify its seq in JSON.
+	if o.OpType != OpWrite {
+		// for Write, ClearRecordIDs and ClearFields were computed
+		// from the records being written, and aren't actually part of the data.
+		if len(o.ClearRecordIDs) != 0 {
+			_, _ = dst.WriteString(`,"record_ids":`)
+			err = codec.appendIDsJSON(dst, o.ClearRecordIDs, codec.keys)
+			if err != nil {
+				return err
+			}
+		}
+		if len(o.ClearFields) != 0 {
+			_, _ = dst.WriteString(`,"fields":`)
+			dst.EncodeStrings(o.ClearFields)
+		}
+	}
+	if len(o.FieldOps) == 0 {
+		_, _ = dst.WriteString(`}`)
+		return
+	}
+	// collect all the fields that have a non-empty set of records. we can
+	// just ignore the others.
+	fieldNames := make([]string, 0, len(o.FieldOps))
+	for field, op := range o.FieldOps {
+		if op != nil && len(op.RecordIDs) != 0 {
+			fieldNames = append(fieldNames, field)
+		}
+	}
+	// nevermind then
+	if len(fieldNames) == 0 {
+		_, _ = dst.WriteString(`}`)
+		return nil
+	}
+	_, _ = dst.WriteString(`,"records":{`)
+
+	// now we have to invert the logic, creating records from
+	// fields with corresponding ops. uh-oh.
+	fieldOps := make([]*FieldOperation, len(o.FieldOps))
+	fieldCodecs := make([]*fieldCodec, len(o.FieldOps))
+	fieldKeys := make([][]string, len(o.FieldOps)) // translated field keys
+	indexes := make([]int, len(o.FieldOps))
+	sort.Slice(fieldNames, func(i, j int) bool { return fieldNames[i] < fieldNames[j] })
+	next := ^uint64(0)
+	var idKeys map[uint64]string
+	if codec.keys != nil {
+		idKeys = make(map[uint64]string)
+	}
+	for i, field := range fieldNames {
+		// populate a parallel slice of field ops so we don't have
+		// to do map lookups for every single piece of data
+		op := o.FieldOps[field]
+		fieldOps[i] = op
+		fc := codec.fields[field]
+		if fc == nil {
+			return fmt.Errorf("unknown field: %q", field)
+		}
+		fieldCodecs[i] = fc
+		if codec.keys != nil {
+			// we'll build a list of keys we need for any records
+			for _, v := range op.RecordIDs {
+				idKeys[v] = ""
+			}
+		}
+		id := op.RecordIDs[0]
+		if id < next {
+			next = id
+		}
+		if fc.keys != nil {
+			thisFieldKeys := make([]string, len(op.RecordIDs))
+			if fc.fieldType == FieldTypeInt {
+				u := make([]uint64, len(op.Signed))
+				for i := range op.Signed {
+					u[i] = uint64(op.Signed[i])
+				}
+				valueKeys, err := fc.keys.TranslateIDs(u...)
+				if err != nil {
+					return err
+				}
+				for j, v := range op.Signed {
+					thisFieldKeys[j] = valueKeys[uint64(v)]
+				}
+			} else {
+				valueKeys, err := fc.keys.TranslateIDs(op.Values...)
+				if err != nil {
+					return err
+				}
+				for j, v := range op.Values {
+					thisFieldKeys[j] = valueKeys[v]
+				}
+			}
+			fieldKeys[i] = thisFieldKeys
+		}
+	}
+	// ... no fieldops actually have any entries. therefore no record will
+	// have any fields set, therefore no records exist...
+	if next == ^uint64(0) {
+		_, _ = dst.WriteString(`}}`)
+		return nil
+	}
+	if codec.keys != nil {
+		idList := make([]uint64, 0, len(idKeys))
+		for k := range idKeys {
+			idList = append(idList, k)
+		}
+		idKeys, err = codec.keys.TranslateIDs(idList...)
+		if err != nil {
+			return err
+		}
+	}
+
+	// next is the ID of a record which exists for at least one field.
+	// we'll recompute it every loop.
+	for next != ^uint64(0) {
+		if codec.keys != nil {
+			dst.EncodeString(idKeys[next])
+		} else {
+			dst.EncodeQuotedUint(next)
+		}
+		_, _ = dst.WriteString(`:{`)
+		current := next
+		next = ^uint64(0)
+		for i, field := range fieldNames {
+			op := fieldOps[i]
+			idx := indexes[i]
+			if idx >= len(op.RecordIDs) {
+				continue
+			}
+			id := op.RecordIDs[idx]
+			if id == current {
+				var j int
+				// count ahead to first index which is either outside the list
+				// or a different id
+				for j = idx; j < len(op.RecordIDs) && op.RecordIDs[j] == id; j++ {
+				}
+				// fmt.Printf("field %s encoding %d-%d (v %d, s %d, k %d)\n",
+				// 	field, idx, j, len(op.Values), len(op.Signed), len(fieldKeys[i]))
+				// print this one, and advance this index to next position
+				dst.EncodeString(field)
+				_, _ = dst.WriteString(`:`)
+				var values []uint64
+				var signed []int64
+				var keys []string
+				if len(op.Values) >= j {
+					values = op.Values[idx:j]
+				}
+				if len(op.Signed) >= j {
+					signed = op.Signed[idx:j]
+				}
+				if len(fieldKeys[i]) >= j {
+					keys = fieldKeys[i][idx:j]
+				}
+				err = fieldCodecs[i].encode(dst, values, signed, keys)
+				if err != nil {
+					return err
+				}
+				_, _ = dst.WriteString(`,`)
+				indexes[i] = j
+				// and we'll fall through to the id < next check, so we
+				// just set id here.
+				if indexes[i] < len(op.RecordIDs) {
+					id = op.RecordIDs[indexes[i]]
+				} else {
+					id = ^uint64(0)
+				}
+			}
+			if id < next {
+				next = id
+			}
+		}
+		// we should always have a trailing comma after any entry, and
+		// if there were no entries we shouldn't have had a list...
+		dst.Truncate(dst.Len() - 1)
+		_, _ = dst.WriteString(`},`)
+	}
+	dst.Truncate(dst.Len() - 1)
+	// close both the records list, and the whole object.
+	_, _ = dst.WriteString(`}}`)
+	// In theory, there's no way for most of these to produce errors,
+	// but just in case, we'll check for an error every operation or so.
+	return dst.Err()
+}
+
 type errFieldNotFound struct {
 	field string
 }
diff --git a/ingest/codec_test.go b/ingest/codec_test.go
index 4490d3729..8605eef5a 100644
--- a/ingest/codec_test.go
+++ b/ingest/codec_test.go
@@ -16,36 +16,481 @@ package ingest
 
 import (
 	"fmt"
+	"sort"
+	"strings"
 	"testing"
 	"time"
 
 	"github.com/molecula/featurebase/v2/shardwidth"
 )
 
-func unusableSampleTranslator(keys ...string) (map[string]uint64, error) {
-	out := make(map[string]uint64, len(keys))
-	for _, key := range keys {
-		out[key] = uint64(len(out)) * 13
+func TestStableTranslator(t *testing.T) {
+	tr := newStableTranslator()
+	m1, err := tr.TranslateKeys("a", "b")
+	if err != nil {
+		t.Fatalf("translation error on initial keys: %v", err)
+	}
+	m2, err := tr.TranslateIDs(m1["a"], m1["b"], 6)
+	if err != nil {
+		t.Fatalf("translation error on reverse lookup: %v", err)
+	}
+	m3, err := tr.TranslateKeys("a", "k-6")
+	if err != nil {
+		t.Fatalf("translation error on new keys: %v", err)
+	}
+	for k, v := range m3 {
+		if m2[v] != k {
+			t.Fatalf("expected round trip to equate %q and %d", k, v)
+		}
 	}
-	return out, nil
 }
 
-func TestSimpleCodec(t *testing.T) {
-	c, _ := NewJSONCodec(nil)
-	_ = c.AddSetField("set", nil)
-	_ = c.AddSetField("setkeys", unusableSampleTranslator)
-	_ = c.AddMutexField("mutex", nil)
-	_ = c.AddMutexField("mutexkeys", unusableSampleTranslator)
-	_ = c.AddTimeQuantumField("tq", nil)
-	_ = c.AddIntField("int", nil)
-	_ = c.AddIntField("intkeys", unusableSampleTranslator)
+func TestMakeCodec(t *testing.T) {
+	codec, _ := NewJSONCodec(nil)
+	err := codec.AddSetField("set", nil)
+	if err != nil {
+		t.Fatalf("unexpected error creating field: %v", err)
+	}
+	err = codec.AddSetField("set", nil)
+	if err == nil {
+		t.Fatalf("expected error creating duplicate field, didn't get it")
+	}
+}
+
+func TestEncode(t *testing.T) {
+	codec, _ := NewJSONCodec(nil)
+	_ = codec.AddSetField("set", nil)
+	_ = codec.AddSetField("setkeys", newStableTranslator())
+	_ = codec.AddMutexField("mutex", nil)
+	_ = codec.AddMutexField("mutexkeys", newStableTranslator())
+	_ = codec.AddTimeQuantumField("tq", nil)
+	_ = codec.AddTimeQuantumField("tqkeys", newStableTranslator())
+	_ = codec.AddIntField("int", nil)
+	_ = codec.AddIntField("intkeys", newStableTranslator())
 	epoch, err := time.Parse("2006-01-02", "2020-01-01")
 	if err != nil {
 		t.Fatalf("can't parse sample epoch time: %v", err)
 	}
-	_ = c.AddTimestampField("ts", time.Millisecond, epoch.Unix()*1000)
-	_ = c.AddDecimalField("dec", 2)
-	_ = c.AddBoolField("bool")
+	_ = codec.AddTimestampField("ts", time.Millisecond, epoch.Unix()*1000)
+	_ = codec.AddDecimalField("dec", 2)
+	_ = codec.AddBoolField("bool")
+
+	codecs := []*JSONCodec{codec}
+
+	// redo all of that, only on a keyed translator
+	codec, _ = NewJSONCodec(newStableTranslator())
+	_ = codec.AddSetField("set", nil)
+	_ = codec.AddSetField("setkeys", newStableTranslator())
+	_ = codec.AddMutexField("mutex", nil)
+	_ = codec.AddMutexField("mutexkeys", newStableTranslator())
+	_ = codec.AddTimeQuantumField("tq", nil)
+	_ = codec.AddTimeQuantumField("tqkeys", newStableTranslator())
+	_ = codec.AddIntField("int", nil)
+	_ = codec.AddIntField("intkeys", newStableTranslator())
+	_ = codec.AddTimestampField("ts", time.Millisecond, epoch.Unix()*1000)
+	_ = codec.AddDecimalField("dec", 2)
+	_ = codec.AddBoolField("bool")
+
+	codecs = append(codecs, codec)
+
+	encodeTests := []*Request{
+		{
+			Ops: []*Operation{
+				{
+					OpType:         OpWrite,
+					ClearRecordIDs: []uint64{0, 1, 2, 3, 5},
+					ClearFields:    []string{"bool", "dec", "int", "intkeys", "mutex", "mutexkeys", "set", "setkeys", "tq", "tqkeys", "ts"},
+					FieldOps: map[string]*FieldOperation{
+						"int": {
+							RecordIDs: []uint64{0, 1},
+							Signed:    []int64{1, -3},
+						},
+						"intkeys": {
+							RecordIDs: []uint64{0, 1},
+							Signed:    []int64{1, 1},
+						},
+						"set": {
+							RecordIDs: []uint64{0, 1, 2, 2},
+							Values:    []uint64{1, 1, 0, 1},
+						},
+						"setkeys": {
+							RecordIDs: []uint64{0, 1, 2, 2},
+							Values:    []uint64{1, 1, 0, 1},
+						},
+						"mutex": {
+							RecordIDs: []uint64{0, 1},
+							Values:    []uint64{1, 2},
+						},
+						"mutexkeys": {
+							RecordIDs: []uint64{0, 1},
+							Values:    []uint64{1, 2},
+						},
+						"tq": {
+							RecordIDs: []uint64{5, 5},
+							Values:    []uint64{8, 9},
+							Signed:    []int64{1234567890e9, 1234567890e9},
+						},
+						"tqkeys": {
+							RecordIDs: []uint64{3, 3},
+							Values:    []uint64{2, 4},
+							Signed:    []int64{1234567890e9, 1234567890e9},
+						},
+						"ts": {
+							RecordIDs: []uint64{0},
+							Signed:    []int64{1},
+						},
+						"bool": {
+							RecordIDs: []uint64{0, 1},
+							Values:    []uint64{0, 1},
+						},
+						"dec": {
+							RecordIDs: []uint64{0, 1, 2},
+							Signed:    []int64{123, -123, 0},
+						},
+					},
+				},
+				{
+					OpType:         OpClear,
+					Seq:            1,
+					ClearRecordIDs: []uint64{6},
+					ClearFields:    []string{"tq"},
+				},
+			},
+		},
+		{
+			// this one needs to get filled in programmatically; see below
+			Ops: []*Operation{
+				{
+					OpType:   OpSet,
+					FieldOps: map[string]*FieldOperation{},
+				},
+			},
+		},
+		{
+			Ops: []*Operation{
+				{
+					OpType: OpRemove,
+					FieldOps: map[string]*FieldOperation{
+						"set": {},
+					},
+				},
+			},
+		},
+	}
+	// and now we populate encodeTests[1] with a larger pool of data
+	const dataSize = 5000
+	shardCount := uint64(600) // shards we want to target
+	passes := uint64(0)
+	recordIDs := make([]uint64, dataSize)
+	values := make([]uint64, dataSize)
+	timeStamps := make([]int64, dataSize)
+	signedValues := make([]int64, dataSize)
+	for i := uint64(0); i < dataSize; i++ {
+		if (i % shardCount) == 0 {
+			passes++
+		}
+		recordIDs[i] = ((i % shardCount) << shardwidth.Exponent) + passes
+		values[i] = (i % 4)
+		timeStamps[i] = int64(1234567890e9 + (i * 100e9))
+		signedValues[i] = (int64(i) % 16) // no negative values because they won't work with keys
+	}
+	// ensure record IDs are sorted, because other stuff might rely on this
+	sort.Slice(recordIDs, func(i, j int) bool { return recordIDs[i] < recordIDs[j] })
+	op := encodeTests[1].Ops[0]
+	op.FieldOps["tq"] = &FieldOperation{
+		RecordIDs: append([]uint64{}, recordIDs...),
+		Values:    append([]uint64{}, values...),
+		Signed:    append([]int64{}, timeStamps...),
+	}
+	op.FieldOps["tqkeys"] = &FieldOperation{
+		RecordIDs: append([]uint64{}, recordIDs...),
+		Values:    values,
+		Signed:    timeStamps,
+	}
+	op.FieldOps["int"] = &FieldOperation{
+		RecordIDs: append([]uint64{}, recordIDs...),
+		Signed:    append([]int64{}, signedValues...),
+	}
+	op.FieldOps["intkeys"] = &FieldOperation{
+		RecordIDs: recordIDs,
+		Signed:    signedValues,
+	}
+	// for sets, we want to shuffle things into fewer shards, and ensure
+	// non-duplication of values within each record, but also have lots
+	// of duplication of record IDs in the low shards
+	recordIDs = make([]uint64, dataSize)
+	values = make([]uint64, dataSize)
+	valuesPerRecord := uint64(5)
+	recordsPerShard := dataSize / valuesPerRecord / 30
+	if recordsPerShard < 1 {
+		recordsPerShard = 1
+	}
+	shard := uint64(0)
+	nextID := uint64(0)
+	nextValue := uint64(0)
+	for i := uint64(0); i < dataSize; i++ {
+		recordIDs[i] = nextID
+		values[i] = nextValue + (i % valuesPerRecord)
+		nextValue++
+		if nextValue == valuesPerRecord {
+			nextValue = 0
+			nextID++
+			if nextID%(1<<shardwidth.Exponent) == recordsPerShard {
+				shard++
+				nextID = (shard << shardwidth.Exponent)
+				if valuesPerRecord > 1 {
+					valuesPerRecord--
+				}
+			}
+		}
+	}
+	sort.Slice(recordIDs, func(i, j int) bool { return recordIDs[i] < recordIDs[j] })
+	op.FieldOps["set"] = &FieldOperation{
+		RecordIDs: append([]uint64{}, recordIDs...),
+		Values:    append([]uint64{}, values...),
+	}
+	op.FieldOps["setkeys"] = &FieldOperation{
+		RecordIDs: recordIDs,
+		Values:    values,
+	}
+	var buf []byte
+	for i, tc := range encodeTests {
+		for _, c := range codecs {
+			data, err := c.AppendBytes(tc, buf[:0])
+			if err != nil {
+				t.Fatalf("encode test %d: error encoding: %v", i, err)
+			}
+			// t.Logf("data:\n%s", data)
+			req, err := c.ParseBytes(data)
+			if err != nil {
+				t.Logf("encode test %d: data:\n%s", i, data)
+				t.Fatalf("encode test %d: error parsing: %v", i, err)
+			}
+			err = req.Compare(tc)
+			if err != nil {
+				t.Logf("encode test %d: data:\n%s", i, data)
+				t.Fatalf("encode test %d: round-trip mismatch: %v", i, err)
+			}
+			data, err = c.AppendBytes(req, buf[:0])
+			if err != nil {
+				t.Fatalf("encode test %d: error encoding: %v", i, err)
+			}
+			// t.Logf("data:\n%s", data)
+			req2, err := c.ParseBytes(data)
+			if err != nil {
+				t.Logf("encode test %d: data:\n%s", i, data)
+				t.Fatalf("encode test %d: error parsing: %v", i, err)
+			}
+			err = req2.Compare(tc)
+			if err != nil {
+				t.Logf("encode test %d: data:\n%s", i, data)
+				t.Fatalf("encode test %d: round-trip mismatch: %v", i, err)
+			}
+		}
+	}
+}
+
+func TestCodecErrors(t *testing.T) {
+	codec, _ := NewJSONCodec(nil)
+	_ = codec.AddSetField("set", nil)
+	_ = codec.AddSetField("setkeys", newStableTranslator())
+	_ = codec.AddMutexField("mutex", nil)
+	_ = codec.AddMutexField("mutexkeys", newStableTranslator())
+	_ = codec.AddTimeQuantumField("tq", nil)
+	_ = codec.AddTimeQuantumField("tqkeys", newStableTranslator())
+	_ = codec.AddIntField("int", nil)
+	_ = codec.AddIntField("intkeys", newStableTranslator())
+	epoch, err := time.Parse("2006-01-02", "2020-01-01")
+	if err != nil {
+		t.Fatalf("can't parse sample epoch time: %v", err)
+	}
+	_ = codec.AddTimestampField("ts", time.Millisecond, epoch.Unix()*1000)
+	_ = codec.AddDecimalField("dec", 2)
+	_ = codec.AddBoolField("bool")
+
+	testCases := []struct {
+		name  string
+		json  []byte
+		error string
+	}{
+		{
+			name:  "no action",
+			json:  []byte(`[{"records":{"0":{"set":[0]}}}]`),
+			error: "action not specified",
+		},
+		{
+			name:  "unknown action",
+			json:  []byte(`[{"action":"yeet","records":{"0":{"set":[0]}}}]`),
+			error: "unknown action",
+		},
+		{
+			name:  "unknown field",
+			json:  []byte(`[{"action":"set","records":{"0":{"settee":[0]}}}]`),
+			error: "field not found",
+		},
+		{
+			name:  "unknown operation field",
+			json:  []byte(`[{"action":"set","yeet":false,"records":{"0":{"set":[0]}}}]`),
+			error: "unknown operation field",
+		},
+		{
+			name:  "expected operation",
+			json:  []byte(`[true]`),
+			error: "expected operation",
+		},
+		{
+			name:  "expecting key",
+			json:  []byte(`[{"action":"set","records":{"0":{"setkeys":0}}}]`),
+			error: "expecting key",
+		},
+		{
+			name:  "invalid int for bool",
+			json:  []byte(`[{"action":"set","records":{"0":{"bool":2}}}]`),
+			error: "boolean should be",
+		},
+		{
+			name:  "invalid number for bool",
+			json:  []byte(`[{"action":"set","records":{"0":{"bool":1.3}}}]`),
+			error: "looks like Number",
+		},
+		{
+			name:  "invalid string for bool",
+			json:  []byte(`[{"action":"set","records":{"0":{"bool":"truly"}}}]`),
+			error: "expecting boolean",
+		},
+		{
+			name:  "nonsense bool",
+			json:  []byte(`[{"action":"set","records":{"0":{"bool":[]}}}]`),
+			error: "boolean should be",
+		},
+		{
+			name:  "expecting numeric value",
+			json:  []byte(`[{"action":"set","records":{"0":{"set":0.1}}}]`),
+			error: "invalid syntax",
+		},
+		{
+			name:  "expecting value",
+			json:  []byte(`[{"action":"set","records":{"0":{"setkeys":true}}}]`),
+			error: "expecting value",
+		},
+		{
+			name:  "expecting array-key",
+			json:  []byte(`[{"action":"set","records":{"0":{"setkeys":[0]}}}]`),
+			error: "expecting key",
+		},
+		{
+			name:  "expecting array-value",
+			json:  []byte(`[{"action":"set","records":{"0":{"setkeys":[true]}}}]`),
+			error: "expecting value",
+		},
+		{
+			name:  "expecting numeric array-value",
+			json:  []byte(`[{"action":"set","records":{"0":{"set":[0.1]}}}]`),
+			error: "invalid syntax",
+		},
+		{
+			name:  "expecting numeric value",
+			json:  []byte(`[{"action":"set","records":{"0":{"int":0.1}}}]`),
+			error: "invalid syntax",
+		},
+		{
+			name:  "expecting int key",
+			json:  []byte(`[{"action":"set","records":{"0":{"intkeys":0}}}]`),
+			error: "expecting string key",
+		},
+		{
+			name:  "expecting int value",
+			json:  []byte(`[{"action":"set","records":{"0":{"int":[0]}}}]`),
+			error: "expecting integer value",
+		},
+		{
+			name:  "expecting string array-value",
+			json:  []byte(`[{"action":"set","records":{"0":{"intkeys":["a"]}}}]`),
+			error: "expecting string key",
+		},
+		{
+			name:  "expecting numeric mutex value",
+			json:  []byte(`[{"action":"set","records":{"0":{"mutex":0.1}}}]`),
+			error: "invalid syntax",
+		},
+		{
+			name:  "expecting mutex key",
+			json:  []byte(`[{"action":"set","records":{"0":{"mutexkeys":0}}}]`),
+			error: "expecting string key",
+		},
+		{
+			name:  "expecting mutex value",
+			json:  []byte(`[{"action":"set","records":{"0":{"mutex":[0]}}}]`),
+			error: "expecting integer value",
+		},
+		{
+			name:  "expecting mutex string value",
+			json:  []byte(`[{"action":"set","records":{"0":{"mutexkeys":["a"]}}}]`),
+			error: "expecting string key",
+		},
+		{
+			name:  "time quantum invalid time",
+			json:  []byte(`[{"action":"set","records":{"0":{"tq":{"time":[],"values":[3]}}}}]`),
+			error: "expecting time",
+		},
+		{
+			name:  "time stamp invalid integer",
+			json:  []byte(`[{"action":"set","records":{"0":{"ts":1.3}}}]`),
+			error: "parsing numeric time",
+		},
+		{
+			name:  "time stamp invalid string",
+			json:  []byte(`[{"action":"set","records":{"0":{"ts":"RFC3339"}}}]`),
+			error: "parsing time",
+		},
+		{
+			name:  "time stamp invalid type",
+			json:  []byte(`[{"action":"set","records":{"0":{"ts":[]}}}]`),
+			error: "expecting time",
+		},
+		{
+			name:  "invalid decimal",
+			json:  []byte(`[{"action":"set","records":{"0":{"dec":[]}}}]`),
+			error: "expecting floating",
+		},
+		{
+			name:  "duplicate record",
+			json:  []byte(`[{"action":"set","records":{"0":{"int":1},"0":{"set":0}}}]`),
+			error: "duplicated in input",
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			req, err := codec.ParseBytes(tc.json)
+			if err == nil {
+				req.Dump(t.Logf)
+				t.Fatalf("expected error like %q, got request instead", tc.error)
+			} else {
+				msg := err.Error()
+				if !strings.Contains(msg, tc.error) {
+					t.Fatalf("expected error like %q, got %q", tc.error, msg)
+				}
+			}
+		})
+	}
+}
+
+func TestSimpleCodec(t *testing.T) {
+	codec, _ := NewJSONCodec(nil)
+	_ = codec.AddSetField("set", nil)
+	_ = codec.AddSetField("setkeys", newStableTranslator())
+	_ = codec.AddMutexField("mutex", nil)
+	_ = codec.AddMutexField("mutexkeys", newStableTranslator())
+	_ = codec.AddTimeQuantumField("tq", nil)
+	_ = codec.AddIntField("int", nil)
+	_ = codec.AddIntField("intkeys", newStableTranslator())
+	epoch, err := time.Parse("2006-01-02", "2020-01-01")
+	if err != nil {
+		t.Fatalf("can't parse sample epoch time: %v", err)
+	}
+	_ = codec.AddTimestampField("ts", time.Millisecond, epoch.Unix()*1000)
+	_ = codec.AddDecimalField("dec", 2)
+	_ = codec.AddBoolField("bool")
 	var nextShard = uint64(1<<shardwidth.Exponent) + 5
 	sampleJson := []byte(fmt.Sprintf(`
 [
@@ -59,8 +504,9 @@ func TestSimpleCodec(t *testing.T) {
           "values": [ 6 ]
         },
 	"dec": 1.02,
-        "int": 3,
+        "int": "3",
 	"mutex": 4,
+	"bool": 1,
 	"mutexkeys": "key-a",
         "intkeys": "key-a"
       },
@@ -70,13 +516,17 @@ func TestSimpleCodec(t *testing.T) {
       },
       "1": {
         "set": [ 2 ],
-	"bool": false,
+	"bool": 0,
         "setkeys": [
           "key-a",
           "key-b"
         ],
         "tq": { "values": [ 3, 4 ] }
-      }
+	},
+	"3":{
+		"bool": "true",
+		"ts": 27
+	}
     }
   },
   {
@@ -92,7 +542,7 @@ func TestSimpleCodec(t *testing.T) {
 	"mutex": 4,
 	"mutexkeys": "key-a",
         "tq": {
-          "time": "2006-01-02T15:04:05.999999999Z",
+          "time": 1234567890,
           "values": 6
         },
 	"ts": "2020-01-01T00:01:00.000000000Z",
@@ -125,135 +575,147 @@ func TestSimpleCodec(t *testing.T) {
   }
 ]
 `, nextShard))
-	var expected = map[uint64][]*Operation{
-		0: {
-			{
-				OpType: OpSet,
-				FieldOps: map[string]*FieldOperation{
-					"mutex": {
-						RecordIDs: []uint64{2},
-						Values:    []uint64{4},
+	var expected = &ShardedRequest{
+		Ops: map[uint64][]*Operation{
+			0: {
+				{
+					OpType: OpSet,
+					FieldOps: map[string]*FieldOperation{
+						"mutex": {
+							RecordIDs: []uint64{2},
+							Values:    []uint64{4},
+						},
+						"set": {
+							RecordIDs: []uint64{1, 2},
+							Values:    []uint64{2, 2},
+						},
+						"tq": {
+							RecordIDs: []uint64{1, 1, 2},
+							Values:    []uint64{3, 4, 6},
+							Signed:    []int64{0, 0, 1136214245999999999},
+						},
+						"mutexkeys": {
+							RecordIDs: []uint64{2},
+							Values:    []uint64{0},
+						},
+						"int": {
+							RecordIDs: []uint64{2},
+							Signed:    []int64{3},
+						},
+						"intkeys": {
+							RecordIDs: []uint64{2},
+							Signed:    []int64{0},
+						},
+						"setkeys": {
+							RecordIDs: []uint64{1, 1},
+							Values:    []uint64{0, 1},
+						},
+						"dec": {
+							RecordIDs: []uint64{2},
+							Signed:    []int64{102},
+						},
+						"bool": {
+							RecordIDs: []uint64{1, 2, 3},
+							Values:    []uint64{0, 1, 1},
+						},
+						"ts": {
+							RecordIDs: []uint64{3},
+							Signed:    []int64{27},
+						},
 					},
-					"set": {
-						RecordIDs: []uint64{1, 2},
-						Values:    []uint64{2, 2},
+				},
+				{
+					OpType:         OpClear,
+					Seq:            1,
+					ClearRecordIDs: []uint64{5, 6, 7},
+					ClearFields:    []string{"tq"},
+				},
+				{
+					OpType:         OpWrite,
+					Seq:            2,
+					ClearRecordIDs: []uint64{3, 4, 5},
+					ClearFields:    []string{"int", "intkeys", "mutex", "mutexkeys", "set", "setkeys", "tq", "ts"},
+					FieldOps: map[string]*FieldOperation{
+						"mutex": {
+							RecordIDs: []uint64{3},
+							Values:    []uint64{4},
+						},
+						"set": {
+							RecordIDs: []uint64{3, 5, 4},
+							Values:    []uint64{2, 2, 3},
+						},
+						"tq": {
+							RecordIDs: []uint64{5, 5, 3},
+							Values:    []uint64{3, 4, 6},
+							Signed:    []int64{0, 0, 1234567890e9},
+						},
+						"mutexkeys": {
+							RecordIDs: []uint64{3},
+							Values:    []uint64{0},
+						},
+						"int": {
+							RecordIDs: []uint64{3},
+							Signed:    []int64{3},
+						},
+						"intkeys": {
+							RecordIDs: []uint64{3},
+							Signed:    []int64{0},
+						},
+						"setkeys": {
+							RecordIDs: []uint64{5, 5},
+							Values:    []uint64{0, 1},
+						},
+						"ts": {
+							RecordIDs: []uint64{3, 4},
+							Signed:    []int64{60000, 1577836860000},
+						},
 					},
-					"tq": {
-						RecordIDs: []uint64{1, 1, 2},
-						Values:    []uint64{3, 4, 6},
-						Signed:    []int64{0, 0, 1136214245999999999},
-					},
-					"mutexkeys": {
-						RecordIDs: []uint64{2},
-						Values:    []uint64{0},
-					},
-					"int": {
-						RecordIDs: []uint64{2},
-						Signed:    []int64{3},
-					},
-					"intkeys": {
-						RecordIDs: []uint64{2},
-						Signed:    []int64{0},
-					},
-					"setkeys": {
-						RecordIDs: []uint64{1, 1},
-						Values:    []uint64{0, 13},
-					},
-					"dec": {
-						RecordIDs: []uint64{2},
-						Signed:    []int64{102},
-					},
-					"bool": {
-						RecordIDs: []uint64{1},
-						Values:    []uint64{0},
+				},
+				{
+					OpType:         OpDelete,
+					Seq:            3,
+					ClearRecordIDs: []uint64{9},
+				},
+				{
+					OpType: OpSet,
+					Seq:    4,
+					FieldOps: map[string]*FieldOperation{
+						"mutex": {
+							RecordIDs: []uint64{2},
+							Values:    []uint64{5},
+						},
+						"mutexkeys": {
+							RecordIDs: []uint64{2},
+							Values:    []uint64{1},
+						},
 					},
 				},
 			},
-			{
-				OpType:         OpClear,
-				ClearRecordIDs: []uint64{5, 6, 7},
-				ClearFields:    []string{"tq"},
-			},
-			{
-				OpType:         OpWrite,
-				ClearRecordIDs: []uint64{3, 4, 5},
-				ClearFields:    []string{"int", "intkeys", "mutex", "mutexkeys", "set", "setkeys", "tq", "ts"},
-				FieldOps: map[string]*FieldOperation{
-					"mutex": {
-						RecordIDs: []uint64{3},
-						Values:    []uint64{4},
-					},
-					"set": {
-						RecordIDs: []uint64{3, 5, 4},
-						Values:    []uint64{2, 2, 3},
-					},
-					"tq": {
-						RecordIDs: []uint64{5, 5, 3},
-						Values:    []uint64{3, 4, 6},
-						Signed:    []int64{0, 0, 1136214245999999999},
-					},
-					"mutexkeys": {
-						RecordIDs: []uint64{3},
-						Values:    []uint64{0},
-					},
-					"int": {
-						RecordIDs: []uint64{3},
-						Signed:    []int64{3},
-					},
-					"intkeys": {
-						RecordIDs: []uint64{3},
-						Signed:    []int64{0},
-					},
-					"setkeys": {
-						RecordIDs: []uint64{5, 5},
-						Values:    []uint64{0, 13},
-					},
-					"ts": {
-						RecordIDs: []uint64{3, 4},
-						Signed:    []int64{60000, 1577836860000},
-					},
-				},
-			},
-			{
-				OpType:         OpDelete,
-				ClearRecordIDs: []uint64{9},
-			},
-			{
-				OpType: OpSet,
-				FieldOps: map[string]*FieldOperation{
-					"mutex": {
-						RecordIDs: []uint64{2},
-						Values:    []uint64{5},
-					},
-					"mutexkeys": {
-						RecordIDs: []uint64{2},
-						Values:    []uint64{13},
-					},
-				},
-			},
-		},
-		1: {
-			{
-				OpType: OpSet,
-				FieldOps: map[string]*FieldOperation{
-					"set": {
-						RecordIDs: []uint64{nextShard},
-						Values:    []uint64{3},
-					},
-					"bool": {
-						RecordIDs: []uint64{nextShard},
-						Values:    []uint64{1},
+			1: {
+				{
+					OpType: OpSet,
+					Seq:    0,
+					FieldOps: map[string]*FieldOperation{
+						"set": {
+							RecordIDs: []uint64{nextShard},
+							Values:    []uint64{3},
+						},
+						"bool": {
+							RecordIDs: []uint64{nextShard},
+							Values:    []uint64{1},
+						},
 					},
 				},
 			},
 		},
 	}
-	req, err := c.ParseBytes(sampleJson)
+	req, err := codec.ParseBytes(sampleJson)
 	if err != nil {
 		t.Fatalf("parsing sample buffer: %v", err)
 	}
 	// req.Dump(t.Logf)
-	sharded, err := req.ByShard()
+	fieldTypes := codec.FieldTypes()
+	sharded, err := req.ByShard(fieldTypes)
 	if err != nil {
 		t.Errorf("sharding err: %v", err)
 	}
@@ -261,15 +723,15 @@ func TestSimpleCodec(t *testing.T) {
 		for i, op := range ops {
 			op.Sort()
 			for field, fop := range op.FieldOps {
-				sorter := fieldTypeSorts[req.FieldTypes[field]]
+				sorter := fieldTypeSorts[fieldTypes[field]]
 				if sorter == nil {
 					sorter = (*FieldOperation).SortByRecords
 				}
 				sorter(fop)
 			}
 			var expectedOp *Operation
-			if i < len(expected[shard]) {
-				expectedOp = expected[shard][i]
+			if i < len(expected.Ops[shard]) {
+				expectedOp = expected.Ops[shard][i]
 			}
 			if err = op.Compare(expectedOp); err != nil {
 				t.Errorf("shard %d, op %d: %v", shard, i, err)
@@ -277,3 +739,253 @@ func TestSimpleCodec(t *testing.T) {
 		}
 	}
 }
+
+func expectEqualFieldOp(t *testing.T, fo1, fo2 *FieldOperation) {
+	if err := fo1.Compare(fo2); err != nil {
+		t.Fatalf("unexpected fieldOp mismatch: %v", err)
+	}
+	if err := fo2.Compare(fo1); err != nil {
+		t.Fatalf("unexpected fieldOp mismatch (inverted): %v", err)
+	}
+}
+
+func expectUnequalFieldOp(t *testing.T, msg string, fo1, fo2 *FieldOperation) {
+	if err := fo1.Compare(fo2); err == nil {
+		t.Fatalf("unexpected fieldOp equality %s", msg)
+	}
+	if err := fo2.Compare(fo1); err == nil {
+		t.Fatalf("unexpected fieldOp equality %s", msg)
+	}
+}
+
+func expectEqualOp(t *testing.T, fo1, fo2 *Operation) {
+	if err := fo1.Compare(fo2); err != nil {
+		t.Fatalf("unexpected Op mismatch: %v", err)
+	}
+	if err := fo2.Compare(fo1); err != nil {
+		t.Fatalf("unexpected Op mismatch (inverted): %v", err)
+	}
+}
+
+func expectUnequalOp(t *testing.T, msg string, fo1, fo2 *Operation) {
+	if err := fo1.Compare(fo2); err == nil {
+		t.Fatalf("unexpected Op equality %s", msg)
+	}
+	if err := fo2.Compare(fo1); err == nil {
+		t.Fatalf("unexpected Op equality %s", msg)
+	}
+}
+
+func expectEqualReq(t *testing.T, fo1, fo2 *Request) {
+	if err := fo1.Compare(fo2); err != nil {
+		t.Fatalf("unexpected Request mismatch: %v", err)
+	}
+	if err := fo2.Compare(fo1); err != nil {
+		t.Fatalf("unexpected Request mismatch (inverted): %v", err)
+	}
+}
+
+func expectUnequalReq(t *testing.T, msg string, fo1, fo2 *Request) {
+	if err := fo1.Compare(fo2); err == nil {
+		t.Fatalf("unexpected Request equality %s", msg)
+	}
+	if err := fo2.Compare(fo1); err == nil {
+		t.Fatalf("unexpected Request equality %s", msg)
+	}
+}
+
+func expectEqualShardedReq(t *testing.T, fo1, fo2 *ShardedRequest) {
+	if err := fo1.Compare(fo2); err != nil {
+		t.Fatalf("unexpected Request mismatch: %v", err)
+	}
+	if err := fo2.Compare(fo1); err != nil {
+		t.Fatalf("unexpected Request mismatch (inverted): %v", err)
+	}
+}
+
+func expectUnequalShardedReq(t *testing.T, msg string, fo1, fo2 *ShardedRequest) {
+	if err := fo1.Compare(fo2); err == nil {
+		t.Fatalf("unexpected Request equality %s", msg)
+	}
+	if err := fo2.Compare(fo1); err == nil {
+		t.Fatalf("unexpected Request equality %s", msg)
+	}
+}
+
+func testCompareFieldOpMutate(t *testing.T, fo1 *FieldOperation) {
+	fo2 := fo1.clone()
+	expectEqualFieldOp(t, fo1, fo2)
+
+	fo1.RecordIDs = fo1.RecordIDs[:1]
+	expectUnequalFieldOp(t, "short record IDs", fo1, fo2)
+	fo1.RecordIDs = fo1.RecordIDs[:2]
+
+	fo1.RecordIDs[1] = 2
+	expectUnequalFieldOp(t, "mismatched record IDs", fo1, fo2)
+	fo1.RecordIDs[1] = fo2.RecordIDs[1]
+
+	if len(fo1.Values) != 0 {
+		fo1.Values = fo1.Values[:1]
+		expectUnequalFieldOp(t, "short values", fo1, fo2)
+		fo1.Values = fo1.Values[:2]
+
+		fo1.Values[1]++
+		expectUnequalFieldOp(t, "mismatched values", fo1, fo2)
+		fo1.Values[1] = fo2.Values[1]
+	}
+
+	if len(fo1.Signed) != 0 {
+		fo1.Signed = fo1.Signed[:1]
+		expectUnequalFieldOp(t, "short signed values", fo1, fo2)
+		fo1.Signed = fo1.Signed[:2]
+
+		fo1.Signed[1]++
+		expectUnequalFieldOp(t, "mismatched signed values", fo1, fo2)
+		fo1.Signed[1] = fo2.Signed[1]
+	}
+}
+
+func TestCompareFieldOperation(t *testing.T) {
+	fo := &FieldOperation{
+		RecordIDs: []uint64{0, 1},
+		Values:    []uint64{0, 1},
+	}
+	testCompareFieldOpMutate(t, fo)
+	fo.Signed = []int64{-3, 5}
+	testCompareFieldOpMutate(t, fo)
+	fo.Values = nil
+	testCompareFieldOpMutate(t, fo)
+
+	// after this, fo has only RecordIDs
+	fo.Signed = nil
+
+	var fNil *FieldOperation
+	expectUnequalFieldOp(t, "nil and non-empty", fo, fNil)
+	fo.RecordIDs = fo.RecordIDs[:0]
+
+	expectEqualFieldOp(t, fo, fNil)
+
+	if err := fNil.Compare(fNil); err != nil {
+		t.Fatalf("expected nil and nil to be equal: %v", err)
+	}
+}
+
+func TestCompareOperation(t *testing.T) {
+	op1 := &Operation{
+		OpType:         OpWrite,
+		ClearRecordIDs: []uint64{0, 1},
+		ClearFields:    []string{"a", "b"},
+		Seq:            1,
+		FieldOps: map[string]*FieldOperation{
+			"c": {
+				RecordIDs: []uint64{0},
+				Values:    []uint64{0},
+			},
+		},
+	}
+	op2 := op1.clone()
+
+	expectEqualOp(t, op1, op2)
+
+	op1.Seq++
+	expectUnequalOp(t, "seq mismatch", op1, op2)
+	op1.Seq = op2.Seq
+
+	op1.OpType++
+	expectUnequalOp(t, "opType mismatch", op1, op2)
+	op1.OpType = op2.OpType
+
+	op1.ClearRecordIDs = op1.ClearRecordIDs[:1]
+	expectUnequalOp(t, "short clearRecordIDs", op1, op2)
+	op1.ClearRecordIDs = op1.ClearRecordIDs[:2]
+
+	op1.ClearRecordIDs[1]++
+	expectUnequalOp(t, "mismatched clearRecordIDs", op1, op2)
+	op1.ClearRecordIDs[1] = op2.ClearRecordIDs[1]
+
+	op1.ClearFields = op1.ClearFields[:1]
+	expectUnequalOp(t, "short clearFields", op1, op2)
+	op1.ClearFields = op1.ClearFields[:2]
+
+	op1.ClearFields[1] = "z"
+	expectUnequalOp(t, "mismatched clearFields", op1, op2)
+	op1.ClearFields[1] = op2.ClearFields[1]
+
+	op1.FieldOps["d"] = op1.FieldOps["c"]
+	expectUnequalOp(t, "extra fieldOp", op1, op2)
+	op2.FieldOps["d"] = op2.FieldOps["c"]
+	expectEqualOp(t, op1, op2)
+	op1.FieldOps["c"].RecordIDs[0]++
+	expectUnequalOp(t, "mismatched fieldOp", op1, op2)
+
+	expectUnequalOp(t, "nil Op", nil, op1)
+	expectEqualOp(t, nil, nil)
+}
+
+func TestCompareRequest(t *testing.T) {
+	r1 := &Request{
+		Ops: []*Operation{
+			{OpType: OpSet},
+			{OpType: OpSet, Seq: 1},
+		},
+	}
+	r2 := &Request{
+		Ops: []*Operation{
+			{OpType: OpSet},
+			{OpType: OpSet, Seq: 1},
+		},
+	}
+	expectEqualReq(t, r1, r2)
+	r1.Ops = r1.Ops[:1]
+	expectUnequalReq(t, "short ops", r1, r2)
+	r1.Ops = r1.Ops[:2]
+
+	r1.Ops[1].Seq = 2
+	expectUnequalReq(t, "ops mismatch", r1, r2)
+	r1.Ops = r1.Ops[:2]
+
+	expectUnequalReq(t, "non-empty and nil", r1, nil)
+
+	r1.Ops = r1.Ops[:0]
+	expectEqualReq(t, r1, nil)
+}
+
+func TestCompareShardedRequest(t *testing.T) {
+	r1 := &ShardedRequest{
+		Ops: map[uint64][]*Operation{
+			1: {
+				{OpType: OpSet},
+				{OpType: OpSet, Seq: 1},
+			},
+			2: {
+				{OpType: OpSet},
+				{OpType: OpSet, Seq: 1},
+			},
+		},
+	}
+	r2 := &ShardedRequest{
+		Ops: map[uint64][]*Operation{
+			1: {
+				{OpType: OpSet},
+				{OpType: OpSet, Seq: 1},
+			},
+			2: {
+				{OpType: OpSet},
+				{OpType: OpSet, Seq: 1},
+			},
+		},
+	}
+	expectEqualShardedReq(t, r1, r2)
+	stash := r1.Ops[2]
+	delete(r1.Ops, 2)
+	expectUnequalShardedReq(t, "short opmap", r1, r2)
+	expectUnequalShardedReq(t, "nil vs nonempty", r1, nil)
+	delete(r1.Ops, 1)
+	expectEqualShardedReq(t, r1, nil)
+	r1.Ops[2] = stash[:1]
+	expectUnequalShardedReq(t, "short ops", r1, r2)
+	r1.Ops[2] = stash
+	r1.Ops[2][1].Seq = 2
+	expectUnequalShardedReq(t, "mismatched ops", r1, r2)
+
+}
diff --git a/ingest/op.go b/ingest/op.go
index b1d6255cd..754d4e4e2 100644
--- a/ingest/op.go
+++ b/ingest/op.go
@@ -15,6 +15,7 @@
 package ingest
 
 import (
+	"errors"
 	"fmt"
 	"math/bits"
 	"sort"
@@ -92,14 +93,22 @@ func ParseOpType(s string) (OpType, error) {
 // record IDs and fields, a Set or Remove will have only FieldOps,
 // and a Write will have both -- populating the record IDs and fields
 // from the fieldops.
+//
+// When we parse an operation, we assign each op a sequential ID
+// within the overall request. We keep these IDs associated with ops
+// when splitting them up across shards, so we can reverse this even
+// if some shards don't get some ops.
 type Operation struct {
 	OpType         OpType
+	Seq            int // sequence position within a chain of ops
 	ClearRecordIDs []uint64
 	ClearFields    []string
 	FieldOps       map[string]*FieldOperation
 }
 
-// Compare reports whether two operations seem to be the same.
+// Compare reports whether two Operations seem to be the same. While
+// a FieldOperation can be "empty" and compare-equal-to a nil FieldOperation,
+// no Operation is considered empty even if it has no FieldOps.
 func (got *Operation) Compare(expected *Operation) error {
 	if got == nil && expected == nil {
 		return nil
@@ -122,27 +131,39 @@ func (got *Operation) Compare(expected *Operation) error {
 			return fmt.Errorf("clear record id %d differs: expected %d, got %d", i, v2, v1)
 		}
 	}
-	if len(got.ClearFields) != len(expected.ClearFields) {
-		return fmt.Errorf("clear field counts differ: expected %d (%q), got %d (%q)", len(expected.ClearFields), expected.ClearFields, len(got.ClearFields), got.ClearFields)
+	// don't assume consistent ordering for the fields, because they're
+	// coming out in arbitrary hash order
+	seenFields := make(map[string]struct{}, len(expected.ClearFields))
+	for _, v1 := range expected.ClearFields {
+		seenFields[v1] = struct{}{}
 	}
-	for i, v1 := range got.ClearFields {
-		v2 := expected.ClearFields[i]
-		if v1 != v2 {
-			return fmt.Errorf("clear field %d differs: expected %q, got %q", i, v2, v1)
+	for _, v2 := range got.ClearFields {
+		if _, ok := seenFields[v2]; !ok {
+			return fmt.Errorf("field %q cleared unexpectedly", v2)
 		}
+		delete(seenFields, v2)
 	}
+	for v1 := range seenFields {
+		return fmt.Errorf("field %q should be cleared but wasn't", v1)
+	}
+	// We check compare even if the op we find on one side is nil, so
+	// a round-trip test will consider a missing op and an empty op to
+	// be interchangeable.
 	for k, fo1 := range got.FieldOps {
 		fo2 := expected.FieldOps[k]
 		if err := fo1.Compare(fo2); err != nil {
 			return fmt.Errorf("field %q mismatch: %w", k, err)
 		}
 	}
-	for k := range expected.FieldOps {
-		_, ok := got.FieldOps[k]
-		if !ok {
-			return fmt.Errorf("expected op for field %q, but none found", k)
+	for k, fo2 := range expected.FieldOps {
+		fo1 := got.FieldOps[k]
+		if err := fo1.Compare(fo2); err != nil {
+			return fmt.Errorf("field %q mismatch: %w", k, err)
 		}
 	}
+	if expected.Seq != got.Seq {
+		return fmt.Errorf("sequence mismatch: expected %d, got %d", expected.Seq, got.Seq)
+	}
 	return nil
 }
 
@@ -172,6 +193,40 @@ func (o *Operation) Sort() {
 	sort.Strings(o.ClearFields)
 }
 
+// clone makes a duplicate of the operation without shared storage
+func (o *Operation) clone() *Operation {
+	o2 := &Operation{
+		OpType:         o.OpType,
+		Seq:            o.Seq,
+		ClearRecordIDs: append([]uint64{}, o.ClearRecordIDs...),
+		ClearFields:    append([]string{}, o.ClearFields...),
+		FieldOps:       make(map[string]*FieldOperation, len(o.FieldOps)),
+	}
+	for k, v := range o.FieldOps {
+		o2.FieldOps[k] = v.clone()
+	}
+	return o2
+}
+
+// merge merges the fieldops of the provided operation into this operation.
+func (o *Operation) merge(o2 *Operation) {
+	o.ClearRecordIDs = append(o.ClearRecordIDs, o2.ClearRecordIDs...)
+	o.ClearFields = append(o.ClearFields, o2.ClearFields...)
+	for f, op := range o2.FieldOps {
+		dst := o.FieldOps[f]
+		if dst == nil {
+			o.FieldOps[f] = op
+			continue
+		}
+		// otherwise append any contents. dst is a pointer-to, so this
+		// updates the thing the map points to, we don't have to write back
+		// into the map.
+		dst.RecordIDs = append(dst.RecordIDs, op.RecordIDs...)
+		dst.Values = append(dst.Values, op.Values...)
+		dst.Signed = append(dst.Signed, op.Signed...)
+	}
+}
+
 type ShardedFieldOperation map[uint64]*FieldOperation
 
 // ByShard() divides the FieldOperation's values up into corresponding chunks
@@ -184,22 +239,14 @@ func (f *FieldOperation) ByShard() ShardedFieldOperation {
 	return f.SortToShards()
 }
 
-// ShardInto puts the shards it finds into a target map.
-func (f *FieldOperation) ShardInto(target ShardedFieldOperation) {
-	shards, ends := shardwidth.FindShards(f.RecordIDs)
-	prev := 0
-	for i, shard := range shards {
-		endIndex := ends[i]
-		subOp := &FieldOperation{RecordIDs: f.RecordIDs[prev:endIndex]}
-		if len(f.Values) > 0 {
-			subOp.Values = f.Values[prev:endIndex]
-		}
-		if len(f.Signed) > 0 {
-			subOp.Signed = f.Signed[prev:endIndex]
-		}
-		target[shard] = subOp
-		prev = endIndex
+// clone makes a duplicate of the operation without shared storage
+func (f *FieldOperation) clone() *FieldOperation {
+	f2 := &FieldOperation{
+		RecordIDs: append([]uint64{}, f.RecordIDs...),
+		Values:    append([]uint64{}, f.Values...),
+		Signed:    append([]int64{}, f.Signed...),
 	}
+	return f2
 }
 
 func ShardIDs(ids []uint64) (out map[uint64][]uint64) {
@@ -349,10 +396,22 @@ func (f *FieldOperation) SortByKeys(keys []uint64) {
 // doesn't help as much as you'd hope. This beats using stdlib sort by about
 // a factor of two for those small N, for larger N we're using the radix sort
 // that calls this.
+//
+// External keys exist only when we are sorting by value, which is to say,
+// when we're using row-oriented formats (set, mutex, time quantum).
+// For int/decimal/timestamp fields, we're sorting by record only.
+// So, if keys is the same as f.RecordIDs, we're looking at an int field
+// or equivalent, so Signed exists and Values doesn't exist.
+// Otherwise, we might be looking at a time quantum field (both exist)
+// or set/mutex (only Values exist).
 func simpleSort(f *FieldOperation, keys []uint64) {
-	if keys != nil {
+	// keys might actually just point to record IDs, in which case, we don't
+	// want to shuffle the corresponding RecordIDs too, because that would just
+	// reverse our swaps. If they're different, we actually need to swap them
+	// both.
+	if &keys[0] != &f.RecordIDs[0] {
 		// sorting by record IDs
-		if f.Values != nil && f.Signed != nil {
+		if f.Values != nil && f.Signed != nil { // time quantum field
 			for i := 1; i < len(keys); i++ {
 				for j := i; j > 0 && keys[j-1] > keys[j]; j-- {
 					keys[j-1], keys[j] = keys[j], keys[j-1]
@@ -362,7 +421,7 @@ func simpleSort(f *FieldOperation, keys []uint64) {
 
 				}
 			}
-		} else if f.Values != nil {
+		} else if f.Values != nil { // set/mutex/bool
 			for i := 1; i < len(keys); i++ {
 				for j := i; j > 0 && keys[j-1] > keys[j]; j-- {
 					keys[j-1], keys[j] = keys[j], keys[j-1]
@@ -371,7 +430,7 @@ func simpleSort(f *FieldOperation, keys []uint64) {
 					f.Values[j-1], f.Values[j] = f.Values[j], f.Values[j-1]
 				}
 			}
-		} else if f.Signed != nil {
+		} else if f.Signed != nil { // can't-happen, we think
 			for i := 1; i < len(keys); i++ {
 				for j := i; j > 0 && keys[j-1] > keys[j]; j-- {
 					keys[j-1], keys[j] = keys[j], keys[j-1]
@@ -379,7 +438,7 @@ func simpleSort(f *FieldOperation, keys []uint64) {
 					f.Signed[j-1], f.Signed[j] = f.Signed[j], f.Signed[j-1]
 				}
 			}
-		} else {
+		} else { // can't-happen, we think
 			for i := 1; i < len(keys); i++ {
 				for j := i; j > 0 && keys[j-1] > keys[j]; j-- {
 					keys[j-1], keys[j] = keys[j], keys[j-1]
@@ -388,7 +447,14 @@ func simpleSort(f *FieldOperation, keys []uint64) {
 			}
 		}
 	} else {
-		if f.Values != nil && f.Signed != nil {
+		if f.Values == nil && f.Signed != nil {
+			for i := 1; i < len(f.RecordIDs); i++ {
+				for j := i; j > 0 && f.RecordIDs[j-1] > f.RecordIDs[j]; j-- {
+					f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1]
+					f.Signed[j-1], f.Signed[j] = f.Signed[j], f.Signed[j-1]
+				}
+			}
+		} else if f.Values != nil && f.Signed != nil { // can't happen, we think
 			for i := 1; i < len(f.RecordIDs); i++ {
 				for j := i; j > 0 && f.RecordIDs[j-1] > f.RecordIDs[j]; j-- {
 					f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1]
@@ -396,22 +462,14 @@ func simpleSort(f *FieldOperation, keys []uint64) {
 					f.Signed[j-1], f.Signed[j] = f.Signed[j], f.Signed[j-1]
 				}
 			}
-		} else if f.Values != nil {
+		} else if f.Values != nil { // only happens during testing
 			for i := 1; i < len(f.RecordIDs); i++ {
 				for j := i; j > 0 && f.RecordIDs[j-1] > f.RecordIDs[j]; j-- {
 					f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1]
 					f.Values[j-1], f.Values[j] = f.Values[j], f.Values[j-1]
 				}
 			}
-		} else if f.Signed != nil {
-			for i := 1; i < len(f.RecordIDs); i++ {
-				for j := i; j > 0 && f.RecordIDs[j-1] > f.RecordIDs[j]; j-- {
-					f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1]
-					f.Signed[j-1], f.Signed[j] = f.Signed[j], f.Signed[j-1]
-				}
-			}
-		} else {
-			// why do we only have record IDs? I don't know
+		} else { // should definitely not happen
 			for i := 1; i < len(f.RecordIDs); i++ {
 				for j := i; j > 0 && f.RecordIDs[j-1] > f.RecordIDs[j]; j-- {
 					f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1]
@@ -493,12 +551,7 @@ func sortPartialByKeys(f *FieldOperation, keys []uint64, shift int) {
 			if end-start > 32 {
 				sortPartialByKeys(&bucketOp, keys[start:end], nextShift)
 			} else {
-				// naive stdlib sort
-				if externalKeys {
-					simpleSort(&bucketOp, keys[start:end])
-				} else {
-					simpleSort(&bucketOp, nil)
-				}
+				simpleSort(&bucketOp, keys[start:end])
 			}
 		}
 	}
@@ -534,16 +587,15 @@ func (got *FieldOperation) Compare(expected *FieldOperation) error {
 		if expected == nil {
 			return nil
 		}
-		if len(expected.RecordIDs) == 0 && len(expected.Values) == 0 && len(expected.Signed) == 0 {
+		// We don't worry about non-empty Values or Signed here, because in theory
+		// RecordIDs are the Source of Truth as to what's in the op.
+		if len(expected.RecordIDs) == 0 {
 			return nil
 		}
 		return fmt.Errorf("expected field operation with %d records, got nil", len(expected.RecordIDs))
 	}
 	if expected == nil {
-		if got == nil {
-			return nil
-		}
-		if len(got.RecordIDs) == 0 && len(got.Values) == 0 && len(got.Signed) == 0 {
+		if len(got.RecordIDs) == 0 {
 			return nil
 		}
 		return fmt.Errorf("expected empty field operation, got %d records", len(got.RecordIDs))
@@ -578,52 +630,6 @@ func (got *FieldOperation) Compare(expected *FieldOperation) error {
 	return nil
 }
 
-func translateUnsignedSlice(target []uint64, mapping []uint64) (err error) {
-	oops := 0
-	for i, v := range target {
-		if v >= uint64(len(mapping)) {
-			oops++
-		} else {
-			target[i] = mapping[v]
-		}
-	}
-	if oops > 0 {
-		return fmt.Errorf("encountered %d out-of-range keys when applying translation mapping", oops)
-	}
-	return nil
-}
-
-// TranslateUnsigned translates keys according to the provided mapping. This
-// is used for sets, mutexes, and time quantums.
-func (op *FieldOperation) TranslateKeys(mapping []uint64) error {
-	return translateUnsignedSlice(op.RecordIDs, mapping)
-}
-
-// TranslateUnsigned translates values according to the provided mapping. This
-// is used for sets, mutexes, and time quantums.
-func (op *FieldOperation) TranslateUnsigned(mapping []uint64) error {
-	return translateUnsignedSlice(op.Values, mapping)
-}
-
-// TranslateSigned translates signed values according to the provided mapping.
-// If we're using this, it's because we're in an integer-type field, which
-// admits using keys for fields, so all key values are actually non-negative,
-// but the field's type still requires values be expressed as signed ints.
-func (op *FieldOperation) TranslateSigned(mapping []uint64) error {
-	oops := 0
-	for i, v := range op.Signed {
-		if v >= int64(len(mapping)) {
-			oops++
-		} else {
-			op.Signed[i] = int64(mapping[v])
-		}
-	}
-	if oops > 0 {
-		return fmt.Errorf("encountered %d out-of-range signed values when applying translation mapping", oops)
-	}
-	return nil
-}
-
 // ShardOperations is a set of Operations associated with a specific shard.
 type ShardOperations struct {
 	Shard uint64
@@ -633,19 +639,17 @@ type ShardOperations struct {
 // Request is a complete ingest request, which may be any combination
 // of operations, which may apply to multiple shards.
 type Request struct {
-	FieldTypes map[string]FieldType
-	Ops        []*Operation
+	Ops []*Operation
 }
 
 // ShardedRequest is an ingest request, split up into individual per-shard
 // operations.
 type ShardedRequest struct {
-	FieldTypes map[string]FieldType
-	Ops        map[uint64][]*Operation
+	Ops map[uint64][]*Operation
 }
 
 // ByShard converts a request into the same request, only sharded.
-func (r *Request) ByShard() (*ShardedRequest, error) {
+func (r *Request) ByShard(fields map[string]FieldType) (*ShardedRequest, error) {
 	if len(r.Ops) == 0 {
 		return &ShardedRequest{Ops: nil}, nil
 	}
@@ -662,12 +666,12 @@ func (r *Request) ByShard() (*ShardedRequest, error) {
 		if op.OpType == OpClear || op.OpType == OpWrite || op.OpType == OpDelete {
 			sharded := ShardIDs(op.ClearRecordIDs)
 			for shard, data := range sharded {
-				shards[shard] = &Operation{OpType: op.OpType, ClearRecordIDs: data, ClearFields: op.ClearFields, FieldOps: map[string]*FieldOperation{}}
+				shards[shard] = &Operation{OpType: op.OpType, Seq: op.Seq, ClearRecordIDs: data, ClearFields: op.ClearFields, FieldOps: map[string]*FieldOperation{}}
 			}
 		}
 		for field, fieldOp := range op.FieldOps {
 			sharded := fieldOp.ByShard()
-			sorter := fieldTypeSorts[r.FieldTypes[field]]
+			sorter := fieldTypeSorts[fields[field]]
 			if sorter == nil {
 				sorter = (*FieldOperation).SortByRecords
 			}
@@ -678,7 +682,7 @@ func (r *Request) ByShard() (*ShardedRequest, error) {
 					if op.OpType == OpWrite {
 						return nil, fmt.Errorf("write operation has field operation data (%d items) for shard %d, but no clear data", len(data.RecordIDs), shard)
 					}
-					shardOp = &Operation{OpType: op.OpType}
+					shardOp = &Operation{OpType: op.OpType, Seq: op.Seq}
 					shards[shard] = shardOp
 					shardOp.FieldOps = map[string]*FieldOperation{field: data}
 				} else {
@@ -697,18 +701,139 @@ func (r *Request) ByShard() (*ShardedRequest, error) {
 	return &ShardedRequest{Ops: req}, nil
 }
 
+// merge combines the components of a sharded request back into a single
+// unsharded request, processing shards in numerical order.
+func (s *ShardedRequest) Merge() *Request {
+	req := &Request{}
+	if s == nil || len(s.Ops) == 0 {
+		return req
+	}
+	shards := make([]uint64, 0, len(s.Ops))
+	for shard := range s.Ops {
+		shards = append(shards, shard)
+	}
+	sort.Slice(shards, func(i, j int) bool { return shards[i] < shards[j] })
+	for _, shard := range shards {
+		ops := s.Ops[shard]
+		for _, op := range ops {
+			var _ *Operation
+			if op.Seq >= len(req.Ops) {
+				// Pad out with nil *Operations to the required length
+				req.Ops = append(req.Ops, make([]*Operation, op.Seq+1-len(req.Ops))...)
+			}
+			if req.Ops[op.Seq] == nil {
+				req.Ops[op.Seq] = op.clone()
+				continue
+			}
+			req.Ops[op.Seq].merge(op)
+		}
+	}
+	return req
+}
+
 func (r *Request) Dump(logf func(string, ...interface{})) {
 	logf("req: %#v", r)
 	for _, op := range r.Ops {
 		logf("op: %#v", op)
 		if len(op.ClearRecordIDs) > 0 {
-			logf("  clearRecordIDs: %d", op.ClearRecordIDs)
+			if len(op.ClearRecordIDs) > 8 {
+				logf("  clearRecordIDs: %d...+%d", op.ClearRecordIDs[:8], len(op.ClearRecordIDs)-8)
+			} else {
+				logf("  clearRecordIDs: %d", op.ClearRecordIDs)
+			}
 		}
 		if len(op.ClearFields) > 0 {
-			logf("  clearFields: %s", op.ClearFields)
+			if len(op.ClearFields) > 8 {
+				logf("  clearFields: %s...+%d", op.ClearFields[:8], len(op.ClearFields)-8)
+			} else {
+				logf("  clearFields: %s", op.ClearFields)
+			}
 		}
 		for field, fieldOp := range op.FieldOps {
-			logf("  field %q: %#v", field, fieldOp)
+			if fieldOp != nil {
+				logf("  field %q: op (%d/%d/%d)", field, len(fieldOp.RecordIDs), len(fieldOp.Values), len(fieldOp.Signed))
+				if len(fieldOp.RecordIDs) > 0 {
+					if len(fieldOp.RecordIDs) > 8 {
+						logf("    records %d...+%d", fieldOp.RecordIDs[:8], len(fieldOp.RecordIDs)-8)
+					} else {
+						logf("    records %d", fieldOp.RecordIDs)
+					}
+				}
+			} else {
+				logf("  field %q: nil op", field)
+			}
 		}
 	}
 }
+
+func (r *Request) Compare(other *Request) error {
+	if other == nil {
+		if r != nil && len(r.Ops) != 0 {
+			return errors.New("non-empty sharded request can't equal empty/nil sharded request")
+		}
+		// empty and nil are allowed
+		return nil
+	}
+	if r == nil {
+		if other != nil && len(other.Ops) != 0 {
+			return errors.New("non-empty sharded request can't equal empty/nil sharded request")
+		}
+		// empty and nil are allowed
+		return nil
+	}
+	ops := r.Ops
+	ops2 := other.Ops
+	if len(ops2) != len(ops) {
+		return fmt.Errorf("expected %d ops, got %d", len(ops), len(ops2))
+	}
+	for i, op := range ops {
+		if err := op.Compare(ops2[i]); err != nil {
+			return fmt.Errorf("op %d: %v", i, err)
+		}
+	}
+	return nil
+}
+
+// Compare checks whether two ShardedRequest objects represent the same
+// data. Empty shards shouldn't have entries in the map in the first place,
+// so we don't accept a nil or 0-length slice of ops as equal to the
+// shard key not existing, but we do accept nil or empty requests as
+// equal to each other.
+func (s *ShardedRequest) Compare(other *ShardedRequest) error {
+	if other == nil {
+		if s != nil && len(s.Ops) != 0 {
+			return errors.New("non-empty sharded request can't equal empty/nil sharded request")
+		}
+		// empty and nil are allowed
+		return nil
+	}
+	if s == nil {
+		if other != nil && len(other.Ops) != 0 {
+			return errors.New("non-empty sharded request can't equal empty/nil sharded request")
+		}
+		// empty and nil are allowed
+		return nil
+	}
+	for shard, ops := range s.Ops {
+		ops2, ok := other.Ops[shard]
+		if !ok {
+			return fmt.Errorf("shard %d missing in other", shard)
+		}
+		if len(ops2) != len(ops) {
+			return fmt.Errorf("shard %d: expected %d ops, got %d", shard, len(ops), len(ops2))
+		}
+		for i, op := range ops {
+			if err := op.Compare(ops2[i]); err != nil {
+				return fmt.Errorf("shard %d, op %d: %v", shard, i, err)
+			}
+		}
+	}
+	if len(other.Ops) != len(s.Ops) {
+		for shard := range other.Ops {
+			if _, ok := s.Ops[shard]; !ok {
+				return fmt.Errorf("shard %d missing in self", shard)
+			}
+		}
+	}
+	return nil
+}
diff --git a/ingest/op_test.go b/ingest/op_test.go
index b0b196e07..db3b5981e 100644
--- a/ingest/op_test.go
+++ b/ingest/op_test.go
@@ -16,7 +16,6 @@ package ingest_test
 
 import (
 	"math/rand"
-	"reflect"
 	"testing"
 
 	"github.com/molecula/featurebase/v2/ingest"
@@ -25,14 +24,14 @@ import (
 
 type opShardingTestCase struct {
 	name   string
-	input  ingest.Request
+	input  *ingest.Request
 	output *ingest.ShardedRequest
 }
 
 var opShardingTestCases = []opShardingTestCase{
 	{
 		name: "sample",
-		input: ingest.Request{
+		input: &ingest.Request{
 			Ops: []*ingest.Operation{
 				{
 					OpType: ingest.OpSet,
@@ -56,6 +55,7 @@ var opShardingTestCases = []opShardingTestCase{
 				},
 				{
 					OpType: ingest.OpRemove,
+					Seq:    1,
 					FieldOps: map[string]*ingest.FieldOperation{
 						"shard0-2": {
 							RecordIDs: []uint64{1, 2<<shardwidth.Exponent + 1},
@@ -82,6 +82,7 @@ var opShardingTestCases = []opShardingTestCase{
 					},
 					{
 						OpType: ingest.OpRemove,
+						Seq:    1,
 						FieldOps: map[string]*ingest.FieldOperation{
 							"shard0-2": {
 								RecordIDs: []uint64{1},
@@ -110,6 +111,7 @@ var opShardingTestCases = []opShardingTestCase{
 				2: {
 					{
 						OpType: ingest.OpRemove,
+						Seq:    1,
 						FieldOps: map[string]*ingest.FieldOperation{
 							"shard0-2": {
 								RecordIDs: []uint64{2<<shardwidth.Exponent + 1},
@@ -122,15 +124,115 @@ var opShardingTestCases = []opShardingTestCase{
 	},
 }
 
-func TestOpSharding(t *testing.T) {
+func TestOpShardingSmall(t *testing.T) {
+	codec, _ := ingest.NewJSONCodec(nil)
+	_ = codec.AddIntField("shard0", nil)
+	_ = codec.AddIntField("shard1", nil)
+	_ = codec.AddIntField("shard0-1", nil)
+	_ = codec.AddIntField("shard0-2", nil)
+
+	fieldTypes := codec.FieldTypes()
 	for _, c := range opShardingTestCases {
-		sharded, err := c.input.ByShard()
+		sharded, err := c.input.ByShard(fieldTypes)
 		if err != nil {
 			t.Errorf("sharding: unexpected error %v", err)
 		}
-		if !reflect.DeepEqual(sharded, c.output) {
-			t.Fatalf("%s: expected %#v, got %#v", c.name, c.output, sharded)
+		if err := sharded.Compare(c.output); err != nil {
+			t.Fatalf("%s: shard: %v", c.name, err)
 		}
+		merged := sharded.Merge()
+		if err := c.input.Compare(merged); err != nil {
+			t.Fatalf("%s: merge: %v", c.name, err)
+		}
+	}
+}
+
+func TestOpShardingLarge(t *testing.T) {
+	codec, err := ingest.NewJSONCodec(nil)
+	if err != nil {
+		t.Fatalf("creating codec: %v", err)
+	}
+	_ = codec.AddSetField("set", nil)
+	_ = codec.AddTimeQuantumField("tq", nil)
+	_ = codec.AddIntField("int", nil)
+
+	// and now we populate encodeTests[1] with a larger pool of data
+	// if this isn't large enough, the scattering across shards means
+	// we coincidentally end up with the time quantum field not getting
+	// tested on simpleSort.
+	const dataSize = 120000
+	shardCount := uint64(300) // shards we want to target
+	passes := uint64(0)
+	recordIDs := make([]uint64, dataSize)
+	values := make([]uint64, dataSize)
+	timeStamps := make([]int64, dataSize)
+	signedValues := make([]int64, dataSize)
+	for i := uint64(0); i < dataSize; i++ {
+		if (i % shardCount) == 0 {
+			passes++
+		}
+		recordIDs[i] = ((i % shardCount) << shardwidth.Exponent) + passes
+		values[i] = (i % 4)
+		timeStamps[i] = int64(1234567890e9 + (i * 100e9))
+		signedValues[i] = (int64(i) % 16) // no negative values because they won't work with keys
+	}
+	op := &ingest.Operation{
+		OpType:   ingest.OpSet,
+		FieldOps: map[string]*ingest.FieldOperation{},
+	}
+	req := &ingest.Request{
+		Ops: []*ingest.Operation{op},
+	}
+	op.FieldOps["tq"] = &ingest.FieldOperation{
+		RecordIDs: append([]uint64{}, recordIDs...),
+		Values:    append([]uint64{}, values...),
+		Signed:    timeStamps,
+	}
+	op.FieldOps["int"] = &ingest.FieldOperation{
+		RecordIDs: recordIDs,
+		Signed:    signedValues,
+	}
+	// for sets, we want to shuffle things into fewer shards, and ensure
+	// non-duplication of values within each record, but also have lots
+	// of duplication of record IDs in the low shards
+	recordIDs = make([]uint64, dataSize)
+	values = make([]uint64, dataSize)
+	valuesPerRecord := uint64(5)
+	recordsPerShard := dataSize / valuesPerRecord / 30
+	if recordsPerShard < 1 {
+		recordsPerShard = 1
+	}
+	shard := uint64(0)
+	nextID := uint64(0)
+	nextValue := uint64(0)
+	for i := uint64(0); i < dataSize; i++ {
+		recordIDs[i] = nextID
+		values[i] = nextValue + (i % valuesPerRecord)
+		nextValue++
+		if nextValue == valuesPerRecord {
+			nextValue = 0
+			nextID++
+			if nextID%(1<<shardwidth.Exponent) == recordsPerShard {
+				shard++
+				nextID = (shard << shardwidth.Exponent)
+				if valuesPerRecord > 1 {
+					valuesPerRecord--
+				}
+			}
+		}
+	}
+	op.FieldOps["set"] = &ingest.FieldOperation{
+		RecordIDs: recordIDs,
+		Values:    values,
+	}
+	fieldTypes := codec.FieldTypes()
+	sharded, err := req.ByShard(fieldTypes)
+	if err != nil {
+		t.Errorf("sharding: unexpected error %v", err)
+	}
+	merged := sharded.Merge()
+	if err := req.Compare(merged); err != nil {
+		t.Fatalf("merge comparison: %v", err)
 	}
 }
 
diff --git a/ingest/translate.go b/ingest/translate.go
index 7ce7ea6dd..42f61e291 100644
--- a/ingest/translate.go
+++ b/ingest/translate.go
@@ -13,3 +13,74 @@
 // limitations under the License.
 
 package ingest
+
+import (
+	"fmt"
+)
+
+// stableTranslator implements a key translator that can be reused and
+// will continue to give the same keys for the same values. Possibly
+// surprisingly, it will invent new keys for IDs it is asked about but
+// hasn't seen. This allows us to give a codec which would use keys on
+// translation a request which contains arbitrary numbers, and request
+// text that would parse into that request.
+type stableTranslator struct {
+	in   map[string]uint64
+	out  map[uint64]string
+	next uint64
+}
+
+func (s *stableTranslator) TranslateKeys(keys ...string) (map[string]uint64, error) {
+	ret := make(map[string]uint64, len(keys))
+	for _, key := range keys {
+		if existing, ok := s.in[key]; ok {
+			ret[key] = existing
+			continue
+		}
+		id := s.next
+		// but what if someone already translated that ID, so now it already
+		// exists?
+		if _, ok := s.out[id]; ok {
+			for k := range s.out {
+				if k > id {
+					id = k
+				}
+			}
+			// one larger than the largest we already have. this could
+			// wrap around, in which case, it's your own fault.
+			id++
+		}
+		s.next = id + 1
+		s.in[key] = id
+		s.out[id] = key
+		ret[key] = id
+	}
+	return ret, nil
+}
+
+func (s *stableTranslator) TranslateIDs(ids ...uint64) (map[uint64]string, error) {
+	ret := make(map[uint64]string, len(ids))
+	for _, id := range ids {
+		if existing, ok := s.out[id]; ok {
+			ret[id] = existing
+			continue
+		}
+		key := fmt.Sprintf("k-%d", id)
+		s.in[key] = id
+		s.out[id] = key
+		ret[id] = key
+		if id >= s.next {
+			s.next = id + 1
+		}
+	}
+	return ret, nil
+}
+
+// newStableTranslator produces a translator which can translate forwards
+// and backwards and invent new things if it needs to. Don't use this.
+func newStableTranslator() *stableTranslator {
+	return &stableTranslator{
+		in:  make(map[string]uint64),
+		out: make(map[uint64]string),
+	}
+}
diff --git a/ingest/translate_test.go b/ingest/translate_test.go
new file mode 100644
index 000000000..78deefda7
--- /dev/null
+++ b/ingest/translate_test.go
@@ -0,0 +1,70 @@
+// Copyright 2021 Molecula Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ingest
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestTranslateReuse(t *testing.T) {
+	// the original stable-translator design had a flaw in that it
+	// assumed that each new ID would always come from a string translation,
+	// never from a key translation, and that they'd show up sequentially.
+	tr := newStableTranslator()
+	orig, err := tr.TranslateIDs(1, 2)
+	tr.next = 1 // intentionally break the translation logic for testing purposes
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	var s [6]string
+	stash := s[:0]
+	for _, v := range orig {
+		stash = append(stash, v)
+	}
+	for i := range s[len(orig):] {
+		stash = append(stash, fmt.Sprintf("key-%d", i))
+	}
+	keys, err := tr.TranslateKeys(stash...)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	ids := make([]uint64, 0, len(keys))
+	for _, v := range keys {
+		ids = append(ids, v)
+	}
+	idMap, err := tr.TranslateIDs(ids...)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	for k, v := range keys {
+		if idMap[v] != k {
+			t.Fatalf("translate mismatch: keys %q->%d, ids %d->%q",
+				k, v, v, idMap[v])
+		}
+	}
+	for id, key := range idMap {
+		if keys[key] != id {
+			t.Fatalf("translate mismatch: ids %d->%q, keys %q->%d",
+				id, key, key, keys[key])
+		}
+	}
+	for id, key := range orig {
+		if keys[key] != id {
+			t.Fatalf("translate mismatch: original ids %d->%q, keys %q->%d",
+				id, key, key, keys[key])
+		}
+	}
+}
diff --git a/ingest/vec.go b/ingest/vec.go
index 082ff8575..14d62cbbc 100644
--- a/ingest/vec.go
+++ b/ingest/vec.go
@@ -19,8 +19,6 @@ import (
 	"reflect"
 	"strconv"
 	"unsafe"
-
-	"github.com/pkg/errors"
 )
 
 // StringTable is a mapping of strings to temporary IDs.
@@ -85,11 +83,8 @@ func (tbl *StringTable) IntID(in []byte) (int64, error) {
 // integers and a translation function from strings to "real" keys, yields
 // a translation/lookup slice. If it cannot translate all the keys, it
 // returns an error.
-//
-// The lookup function corresponds to the FindKeys/CreateKeys methods of
-// featurebase translators, by an AMAZING coincidence.
-func MapForStringTable(tbl *StringTable, lookup func(...string) (map[string]uint64, error)) ([]uint64, error) {
-	lookedUp, err := lookup(tbl.names...)
+func (tbl *StringTable) MakeIDMap(keys KeyTranslator) ([]uint64, error) {
+	lookedUp, err := keys.TranslateKeys(tbl.names...)
 	if err != nil {
 		return nil, err
 	}
@@ -103,22 +98,38 @@ func MapForStringTable(tbl *StringTable, lookup func(...string) (map[string]uint
 	return out, nil
 }
 
-// TimeFormatForUnit returns the time transfer format (between the update encoder and the update applier) with appropriate resolution for a quantum unit.
-func TimeFormatForUnit(unit rune) string {
-	switch unit {
-	case 'Y':
-		return "2006"
-	case 'M':
-		return "200601"
-	case 'D':
-		return "20060102"
-	case 'H':
-		return "2006010203"
-	default:
-		panic(errors.Errorf("invalid quantum unit: %q", unit))
+// translateSigned replaces values from 0 to len(mapping)-1 with the
+// elements of mapping. It yields an error if any values aren't
+// mapped.
+func translateSigned(mapping []uint64, values []int64) error {
+	oops := 0
+	for i, v := range values {
+		if v >= int64(len(mapping)) {
+			oops++
+		} else {
+			values[i] = int64(mapping[v])
+		}
 	}
+	if oops > 0 {
+		return fmt.Errorf("encountered %d out-of-range signed values when applying translation mapping", oops)
+	}
+	return nil
 }
 
-// TODO: bool
-
-// TODO: timestamp (just sugar on top of IntVector)
+// translateUnsigned replaces values from 0 to len(mapping)-1 with the
+// elements of mapping. It yields an error if any values aren't
+// mapped.
+func translateUnsigned(mapping []uint64, values []uint64) error {
+	oops := 0
+	for i, v := range values {
+		if v >= uint64(len(mapping)) {
+			oops++
+		} else {
+			values[i] = mapping[v]
+		}
+	}
+	if oops > 0 {
+		return fmt.Errorf("encountered %d out-of-range signed values when applying translation mapping", oops)
+	}
+	return nil
+}
diff --git a/ingest/vec_test.go b/ingest/vec_test.go
new file mode 100644
index 000000000..45a103645
--- /dev/null
+++ b/ingest/vec_test.go
@@ -0,0 +1,114 @@
+// Copyright 2021 Molecula Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ingest
+
+import (
+	"errors"
+	"testing"
+)
+
+type badTranslator struct{}
+
+func (b badTranslator) TranslateKeys(keys ...string) (map[string]uint64, error) {
+	if len(keys) == 0 {
+		return nil, errors.New("no keys")
+	}
+	m := make(map[string]uint64)
+	skip := true
+	for i, k := range keys {
+		if skip {
+			skip = false
+			continue
+		}
+		m[k] = uint64(i)
+	}
+	out := make([]uint64, len(keys)-1)
+	for i := range out {
+		out[i] = uint64(i)
+	}
+	return nil, nil
+}
+
+func (b badTranslator) TranslateIDs(...uint64) (map[uint64]string, error) {
+	return nil, nil
+}
+
+func TestStringTableErrors(t *testing.T) {
+	tbl := NewStringTable()
+	btr := badTranslator{}
+	_, keyErr := tbl.MakeIDMap(btr)
+	if keyErr == nil {
+		t.Fatalf("expected error passed up from failed translate, didn't get it")
+	}
+	a1, err := tbl.ID([]byte("a"))
+	if err != nil {
+		t.Fatalf("getting translation for key: %v", err)
+	}
+	b1, err := tbl.ID([]byte("b"))
+	if err != nil {
+		t.Fatalf("getting translation for key: %v", err)
+	}
+	_, keyErr = tbl.MakeIDMap(btr)
+	if keyErr == nil {
+		t.Fatalf("expected error for short translate, didn't get it")
+	}
+	tr := newStableTranslator()
+	_, err = tr.TranslateKeys("c", "d")
+	if err != nil {
+		t.Fatalf("translating stray keys: %v", err)
+	}
+	m, err := tbl.MakeIDMap(tr)
+	if err != nil {
+		t.Fatalf("creating lookup: %v", err)
+	}
+	var y = []uint64{a1, b1}
+	err = translateUnsigned(m, y)
+	if err != nil {
+		t.Fatalf("unexpected unsigned translation error: %v", err)
+	}
+	trResults, err := tr.TranslateKeys("a", "b")
+	if err != nil {
+		t.Fatalf("unexpected translation error: %v", err)
+	}
+	if y[0] != trResults["a"] {
+		t.Fatalf("expected %d, got %d", trResults["a"], y[0])
+	}
+	if y[1] != trResults["b"] {
+		t.Fatalf("expected %d, got %d", trResults["b"], y[1])
+	}
+	y[0] = a1
+	y[1] = (a1 + b1 + 1) // assumed not to be any of them
+	err = translateUnsigned(m, y)
+	if err == nil {
+		t.Fatalf("no error from translating invalid table")
+	}
+	z := []int64{int64(a1), int64(b1)}
+	err = translateSigned(m, z)
+	if err != nil {
+		t.Fatalf("unexpected unsigned translation error: %v", err)
+	}
+	if uint64(z[0]) != trResults["a"] {
+		t.Fatalf("expected %d, got %d", trResults["a"], z[0])
+	}
+	if uint64(z[1]) != trResults["b"] {
+		t.Fatalf("expected %d, got %d", trResults["b"], z[1])
+	}
+	z[0] = int64(a1)
+	z[1] = int64(a1 + b1 + 1) // assumed not to be any of them
+	err = translateSigned(m, z)
+	if err == nil {
+		t.Fatalf("no error from translating invalid table")
+	}
+}
diff --git a/ingest_test.go b/ingest_test.go
index 649f5dd16..5cd4beac4 100644
--- a/ingest_test.go
+++ b/ingest_test.go
@@ -26,9 +26,7 @@ import (
 	"testing"
 
 	pilosa "github.com/molecula/featurebase/v2"
-	"github.com/molecula/featurebase/v2/http"
 	"github.com/molecula/featurebase/v2/ingest"
-	"github.com/molecula/featurebase/v2/server"
 	"github.com/molecula/featurebase/v2/test"
 	"github.com/pkg/errors"
 )
@@ -448,14 +446,7 @@ func TestIngestTestcases(t *testing.T) {
 	}
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	c := test.MustRunCluster(t, 1,
-		[]server.CommandOption{
-			server.OptCommandServerOptions(
-				pilosa.OptServerNodeID("node0"),
-				pilosa.OptServerClusterHasher(&offsetModHasher{}),
-				pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
-			)},
-	)
+	c := test.MustRunCluster(t, 3)
 	defer c.Close()
 
 	coord := c.GetPrimary()
diff --git a/translate.go b/translate.go
index 738fc1686..f449d75be 100644
--- a/translate.go
+++ b/translate.go
@@ -23,6 +23,7 @@ import (
 	"sort"
 	"sync"
 
+	"github.com/molecula/featurebase/v2/ingest"
 	"github.com/molecula/featurebase/v2/topology"
 	"github.com/pkg/errors"
 )
@@ -98,6 +99,39 @@ type TranslateStore interface { // TODO: refactor this interface; readonly shoul
 	ReadFrom(io.Reader) (int64, error)
 }
 
+// This implements ingest's key translator interface, which differs
+// slightly because we want to be able to do fast lookups on arbitrary
+// IDs which are not necessarily contiguous small values, so the []string
+// from TranslateIDs isn't a good fit.
+type ingestKeyTranslator struct {
+	store TranslateStore
+}
+
+var _ ingest.KeyTranslator = &ingestKeyTranslator{}
+
+func (i ingestKeyTranslator) TranslateKeys(keys ...string) (map[string]uint64, error) {
+	return i.store.CreateKeys(keys...)
+}
+
+func (i ingestKeyTranslator) TranslateIDs(ids ...uint64) (map[uint64]string, error) {
+	keys, err := i.store.TranslateIDs(ids)
+	if err != nil {
+		return nil, err
+	}
+	if len(keys) != len(ids) {
+		return nil, fmt.Errorf("translating %d id(s), got %d key(s)", len(ids), len(keys))
+	}
+	out := make(map[uint64]string, len(keys))
+	for i, id := range ids {
+		out[id] = keys[i]
+	}
+	return out, nil
+}
+
+func newIngestKeyTranslatorFromStore(s TranslateStore) *ingestKeyTranslator {
+	return &ingestKeyTranslator{store: s}
+}
+
 // TranslatorSummary is returned, for example from the boltdb string key translators,
 // by calling ComputeTranslatorSummary(). Non-boltdb mocks, etc no-op that method.
 type TranslatorSummary struct {

From 610c4ed6cbd8b3d4d3cc2a0bcd4704151f8fc1fd Mon Sep 17 00:00:00 2001
From: Seebs <seebs@molecula.com>
Date: Fri, 10 Sep 2021 12:25:43 -0500
Subject: [PATCH 40/66] introduce protobuf types for ingest ops

We add a new protobuf type. Also, protoc changed slightly and remade
some tests, in a way which should have no effects but makes the code
*very* slightly cleaner.

This introduces the first testing code in encoding/proto (whoops)
so that scaffolding is a first draft; if you're looking at this code
and the design is a problem go ahead and fix it.

The purpose of this is to verify that we're actually covering all
the branches in the ingest.ShardedRequest and pb.ShardedIngestRequest
message conversions. (Except the top-level one for a nil request,
which isn't checked by this.)

The coverage report doesn't actually include coverage for the ingest
code, though, so we haven't actually properly tested Compare.
Baby steps!
---
 encoding/proto/proto.go      |    3 +
 encoding/proto/proto_test.go |  149 +++
 pb/private.pb.go             | 1852 +++++++++++++++++++++++++++++-----
 pb/private.proto             |   23 +-
 pb/public.pb.go              |  170 +---
 5 files changed, 1813 insertions(+), 384 deletions(-)
 create mode 100644 encoding/proto/proto_test.go

diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go
index d765a8aff..9a8a262a0 100644
--- a/encoding/proto/proto.go
+++ b/encoding/proto/proto.go
@@ -978,6 +978,9 @@ func (s Serializer) encodeShardIngestOperation(op *ingest.Operation) *pb.ShardIn
 		FieldOps:       make(map[string]*pb.FieldOperation, len(op.FieldOps)),
 	}
 	for k, v := range op.FieldOps {
+		if v == nil {
+			continue
+		}
 		out.FieldOps[k] = &pb.FieldOperation{
 			RecordIDs: v.RecordIDs,
 			Values:    v.Values,
diff --git a/encoding/proto/proto_test.go b/encoding/proto/proto_test.go
new file mode 100644
index 000000000..7ea635881
--- /dev/null
+++ b/encoding/proto/proto_test.go
@@ -0,0 +1,149 @@
+// Copyright 2021 Molecula Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package proto
+
+import (
+	"errors"
+	"reflect"
+	"testing"
+
+	"github.com/molecula/featurebase/v2"
+	"github.com/molecula/featurebase/v2/ingest"
+)
+
+func testOneRoundTrip(t *testing.T, s pilosa.Serializer, obj pilosa.Message, expectedMarshalErr error, expectedUnmarshalErr error, expectedMismatchErr error) {
+	repr, err := s.Marshal(obj)
+	if err != nil {
+		if expectedMarshalErr == nil {
+			t.Fatalf("unexpected marshalling error %q", err.Error())
+		}
+		if err.Error() != expectedMarshalErr.Error() {
+			t.Fatalf("expecting marshalling error %q, got %q", expectedMarshalErr.Error(), err.Error())
+		}
+	} else {
+		if expectedMarshalErr != nil {
+			t.Fatalf("expected marshalling error %q, got no error", expectedMarshalErr.Error())
+		}
+	}
+
+	obj2 := reflect.New(reflect.TypeOf(obj).Elem()).Interface()
+	err = s.Unmarshal(repr, obj2)
+	if err != nil {
+		if expectedUnmarshalErr == nil {
+			t.Fatalf("unexpected unmarshalling error %q", err.Error())
+		}
+		if err.Error() != expectedUnmarshalErr.Error() {
+			t.Fatalf("expecting unmarshalling error %q, got %q", expectedUnmarshalErr.Error(), err.Error())
+		}
+	} else {
+		if expectedUnmarshalErr != nil {
+			t.Fatalf("expected unmarshalling error %q, got no error", expectedUnmarshalErr.Error())
+		}
+	}
+	switch real := obj.(type) {
+	case *ingest.ShardedRequest:
+		real2 := obj2.(*ingest.ShardedRequest)
+		err := real.Compare(real2)
+		if err != nil {
+			if expectedMismatchErr == nil {
+				t.Fatalf("unexpected compare error %q", err.Error())
+			}
+			if err.Error() != expectedMismatchErr.Error() {
+				t.Fatalf("expecting compare error %q, got %q", expectedMismatchErr.Error(), err.Error())
+			}
+		} else {
+			if expectedMismatchErr != nil {
+				t.Fatalf("expected compare error %q, got no error", expectedMismatchErr.Error())
+			}
+		}
+	default:
+		if !reflect.DeepEqual(obj, obj2) {
+			t.Fatalf("serialization round trip failed for %T:\nexpected %#v\ngot %#v", obj, obj, obj2)
+		}
+	}
+}
+
+type shardedIngestRequestTest struct {
+	req *ingest.ShardedRequest
+	err error
+}
+
+var shardedIngestRequestTestcases = []shardedIngestRequestTest{
+	{
+		req: &ingest.ShardedRequest{
+			Ops: map[uint64][]*ingest.Operation{
+				1: {
+					{
+						OpType:         ingest.OpWrite,
+						ClearFields:    []string{"clearField", "clearField2"},
+						ClearRecordIDs: []uint64{1, 7, 9},
+						FieldOps: map[string]*ingest.FieldOperation{
+							"writeAll": {
+								RecordIDs: []uint64{3, 6, 8},
+								Values:    []uint64{0, 17, 34},
+								Signed:    []int64{-9, 23, 17},
+							},
+							"writeValues": {
+								RecordIDs: []uint64{3, 6, 8},
+								Values:    []uint64{0, 17, 34},
+							},
+							"writeSigned": {
+								RecordIDs: []uint64{3, 6, 8},
+								Signed:    []int64{-9, 23, 17},
+							},
+						},
+					},
+					{
+						OpType: ingest.OpSet,
+						FieldOps: map[string]*ingest.FieldOperation{
+							"foo": nil,
+						},
+					},
+				},
+				2: {},
+				3: nil,
+			},
+		},
+	},
+	{
+		req: &ingest.ShardedRequest{
+			Ops: map[uint64][]*ingest.Operation{
+				1: {
+					{
+						OpType: ingest.OpWrite,
+						FieldOps: map[string]*ingest.FieldOperation{
+							"writeAll": {},
+						},
+					},
+					{
+						OpType: ingest.OpSet,
+						FieldOps: map[string]*ingest.FieldOperation{
+							"foo": nil,
+						},
+					},
+					nil,
+				},
+			},
+		},
+		err: errors.New("shard 1: expected 3 ops, got 2"),
+	},
+}
+
+func TestIngestRoundTrip(t *testing.T) {
+	for _, tc := range shardedIngestRequestTestcases {
+		t.Logf("next case")
+		testOneRoundTrip(t, DefaultSerializer, tc.req, nil, nil, tc.err)
+	}
+}
diff --git a/pb/private.pb.go b/pb/private.pb.go
index 54366c2cb..3a2807420 100644
--- a/pb/private.pb.go
+++ b/pb/private.pb.go
@@ -2479,6 +2479,234 @@ func (m *ResizeNodeMessage) GetAction() string {
 	return ""
 }
 
+type FieldOperation struct {
+	RecordIDs            []uint64 `protobuf:"varint,1,rep,packed,name=RecordIDs,proto3" json:"RecordIDs,omitempty"`
+	Values               []uint64 `protobuf:"varint,2,rep,packed,name=Values,proto3" json:"Values,omitempty"`
+	Signed               []int64  `protobuf:"varint,3,rep,packed,name=Signed,proto3" json:"Signed,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *FieldOperation) Reset()         { *m = FieldOperation{} }
+func (m *FieldOperation) String() string { return proto.CompactTextString(m) }
+func (*FieldOperation) ProtoMessage()    {}
+func (*FieldOperation) Descriptor() ([]byte, []int) {
+	return fileDescriptor_d2a91b51c7bdc125, []int{39}
+}
+func (m *FieldOperation) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *FieldOperation) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_FieldOperation.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *FieldOperation) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_FieldOperation.Merge(m, src)
+}
+func (m *FieldOperation) XXX_Size() int {
+	return m.Size()
+}
+func (m *FieldOperation) XXX_DiscardUnknown() {
+	xxx_messageInfo_FieldOperation.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_FieldOperation proto.InternalMessageInfo
+
+func (m *FieldOperation) GetRecordIDs() []uint64 {
+	if m != nil {
+		return m.RecordIDs
+	}
+	return nil
+}
+
+func (m *FieldOperation) GetValues() []uint64 {
+	if m != nil {
+		return m.Values
+	}
+	return nil
+}
+
+func (m *FieldOperation) GetSigned() []int64 {
+	if m != nil {
+		return m.Signed
+	}
+	return nil
+}
+
+type ShardIngestOperation struct {
+	OpType               string                     `protobuf:"bytes,1,opt,name=OpType,proto3" json:"OpType,omitempty"`
+	ClearRecordIDs       []uint64                   `protobuf:"varint,2,rep,packed,name=ClearRecordIDs,proto3" json:"ClearRecordIDs,omitempty"`
+	ClearFields          []string                   `protobuf:"bytes,3,rep,name=ClearFields,proto3" json:"ClearFields,omitempty"`
+	FieldOps             map[string]*FieldOperation `protobuf:"bytes,4,rep,name=FieldOps,proto3" json:"FieldOps,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+	XXX_NoUnkeyedLiteral struct{}                   `json:"-"`
+	XXX_unrecognized     []byte                     `json:"-"`
+	XXX_sizecache        int32                      `json:"-"`
+}
+
+func (m *ShardIngestOperation) Reset()         { *m = ShardIngestOperation{} }
+func (m *ShardIngestOperation) String() string { return proto.CompactTextString(m) }
+func (*ShardIngestOperation) ProtoMessage()    {}
+func (*ShardIngestOperation) Descriptor() ([]byte, []int) {
+	return fileDescriptor_d2a91b51c7bdc125, []int{40}
+}
+func (m *ShardIngestOperation) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *ShardIngestOperation) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_ShardIngestOperation.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *ShardIngestOperation) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_ShardIngestOperation.Merge(m, src)
+}
+func (m *ShardIngestOperation) XXX_Size() int {
+	return m.Size()
+}
+func (m *ShardIngestOperation) XXX_DiscardUnknown() {
+	xxx_messageInfo_ShardIngestOperation.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ShardIngestOperation proto.InternalMessageInfo
+
+func (m *ShardIngestOperation) GetOpType() string {
+	if m != nil {
+		return m.OpType
+	}
+	return ""
+}
+
+func (m *ShardIngestOperation) GetClearRecordIDs() []uint64 {
+	if m != nil {
+		return m.ClearRecordIDs
+	}
+	return nil
+}
+
+func (m *ShardIngestOperation) GetClearFields() []string {
+	if m != nil {
+		return m.ClearFields
+	}
+	return nil
+}
+
+func (m *ShardIngestOperation) GetFieldOps() map[string]*FieldOperation {
+	if m != nil {
+		return m.FieldOps
+	}
+	return nil
+}
+
+type ShardIngestOperations struct {
+	Ops                  []*ShardIngestOperation `protobuf:"bytes,1,rep,name=Ops,proto3" json:"Ops,omitempty"`
+	XXX_NoUnkeyedLiteral struct{}                `json:"-"`
+	XXX_unrecognized     []byte                  `json:"-"`
+	XXX_sizecache        int32                   `json:"-"`
+}
+
+func (m *ShardIngestOperations) Reset()         { *m = ShardIngestOperations{} }
+func (m *ShardIngestOperations) String() string { return proto.CompactTextString(m) }
+func (*ShardIngestOperations) ProtoMessage()    {}
+func (*ShardIngestOperations) Descriptor() ([]byte, []int) {
+	return fileDescriptor_d2a91b51c7bdc125, []int{41}
+}
+func (m *ShardIngestOperations) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *ShardIngestOperations) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_ShardIngestOperations.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *ShardIngestOperations) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_ShardIngestOperations.Merge(m, src)
+}
+func (m *ShardIngestOperations) XXX_Size() int {
+	return m.Size()
+}
+func (m *ShardIngestOperations) XXX_DiscardUnknown() {
+	xxx_messageInfo_ShardIngestOperations.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ShardIngestOperations proto.InternalMessageInfo
+
+func (m *ShardIngestOperations) GetOps() []*ShardIngestOperation {
+	if m != nil {
+		return m.Ops
+	}
+	return nil
+}
+
+type ShardedIngestRequest struct {
+	Ops                  map[uint64]*ShardIngestOperations `protobuf:"bytes,1,rep,name=Ops,proto3" json:"Ops,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+	XXX_NoUnkeyedLiteral struct{}                          `json:"-"`
+	XXX_unrecognized     []byte                            `json:"-"`
+	XXX_sizecache        int32                             `json:"-"`
+}
+
+func (m *ShardedIngestRequest) Reset()         { *m = ShardedIngestRequest{} }
+func (m *ShardedIngestRequest) String() string { return proto.CompactTextString(m) }
+func (*ShardedIngestRequest) ProtoMessage()    {}
+func (*ShardedIngestRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_d2a91b51c7bdc125, []int{42}
+}
+func (m *ShardedIngestRequest) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *ShardedIngestRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_ShardedIngestRequest.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *ShardedIngestRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_ShardedIngestRequest.Merge(m, src)
+}
+func (m *ShardedIngestRequest) XXX_Size() int {
+	return m.Size()
+}
+func (m *ShardedIngestRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_ShardedIngestRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ShardedIngestRequest proto.InternalMessageInfo
+
+func (m *ShardedIngestRequest) GetOps() map[uint64]*ShardIngestOperations {
+	if m != nil {
+		return m.Ops
+	}
+	return nil
+}
+
 func init() {
 	proto.RegisterType((*IndexMeta)(nil), "pb.IndexMeta")
 	proto.RegisterType((*FieldOptions)(nil), "pb.FieldOptions")
@@ -2520,103 +2748,121 @@ func init() {
 	proto.RegisterType((*TransactionStats)(nil), "pb.TransactionStats")
 	proto.RegisterType((*ResizeAbortMessage)(nil), "pb.ResizeAbortMessage")
 	proto.RegisterType((*ResizeNodeMessage)(nil), "pb.ResizeNodeMessage")
+	proto.RegisterType((*FieldOperation)(nil), "pb.FieldOperation")
+	proto.RegisterType((*ShardIngestOperation)(nil), "pb.ShardIngestOperation")
+	proto.RegisterMapType((map[string]*FieldOperation)(nil), "pb.ShardIngestOperation.FieldOpsEntry")
+	proto.RegisterType((*ShardIngestOperations)(nil), "pb.ShardIngestOperations")
+	proto.RegisterType((*ShardedIngestRequest)(nil), "pb.ShardedIngestRequest")
+	proto.RegisterMapType((map[uint64]*ShardIngestOperations)(nil), "pb.ShardedIngestRequest.OpsEntry")
 }
 
 func init() { proto.RegisterFile("private.proto", fileDescriptor_d2a91b51c7bdc125) }
 
 var fileDescriptor_d2a91b51c7bdc125 = []byte{
-	// 1450 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x5d, 0x6f, 0x1b, 0x45,
-	0x17, 0x7e, 0xf7, 0xc3, 0xb1, 0x7d, 0x1c, 0x27, 0xce, 0x34, 0xea, 0xbb, 0xfd, 0x78, 0x23, 0x77,
-	0x5e, 0x44, 0x43, 0x25, 0x22, 0x51, 0x2e, 0x8a, 0xe0, 0xa6, 0x49, 0x9c, 0x16, 0x53, 0xd2, 0x86,
-	0x71, 0xda, 0x5b, 0x34, 0x5e, 0x8f, 0x9a, 0x55, 0xd6, 0xbb, 0x66, 0x3f, 0x52, 0xbb, 0x17, 0x48,
-	0x20, 0x10, 0xfc, 0x04, 0x7e, 0x06, 0x37, 0xfc, 0x07, 0x6e, 0x90, 0xf8, 0x09, 0xa8, 0xfc, 0x11,
-	0x34, 0x67, 0x66, 0x76, 0xd7, 0xae, 0x5b, 0x43, 0xc4, 0xdd, 0x9e, 0xe7, 0xcc, 0x9c, 0xf3, 0x9c,
-	0x8f, 0x39, 0x33, 0x0b, 0xed, 0x49, 0x12, 0x5c, 0xf0, 0x4c, 0xec, 0x4d, 0x92, 0x38, 0x8b, 0x89,
-	0x3d, 0x19, 0x5e, 0x5f, 0x9f, 0xe4, 0xc3, 0x30, 0xf0, 0x15, 0x42, 0x1f, 0x42, 0xb3, 0x1f, 0x8d,
-	0xc4, 0xf4, 0x58, 0x64, 0x9c, 0x10, 0x70, 0x1f, 0x89, 0x59, 0xea, 0x39, 0x5d, 0x6b, 0xb7, 0xc1,
-	0xf0, 0x9b, 0xbc, 0x0b, 0x1b, 0xa7, 0x09, 0xf7, 0xcf, 0x8f, 0xa6, 0x41, 0x9a, 0x89, 0xc8, 0x17,
-	0x9e, 0x8b, 0xda, 0x05, 0x94, 0xfe, 0xec, 0xc0, 0xfa, 0x83, 0x40, 0x84, 0xa3, 0x27, 0x93, 0x2c,
-	0x88, 0xa3, 0x54, 0x1a, 0x3b, 0x9d, 0x4d, 0x84, 0xd7, 0xe8, 0x5a, 0xbb, 0x4d, 0x86, 0xdf, 0xe4,
-	0x26, 0x34, 0x0f, 0xb9, 0x7f, 0x26, 0x50, 0xe1, 0xa0, 0xa2, 0x04, 0x0a, 0xed, 0x20, 0x78, 0xa9,
-	0xbc, 0xb4, 0x59, 0x09, 0x90, 0x2e, 0xb4, 0x4e, 0x83, 0xb1, 0xf8, 0x22, 0xe7, 0x51, 0x96, 0x8f,
-	0xbd, 0x1a, 0xee, 0xae, 0x42, 0xe4, 0x2a, 0xac, 0x3d, 0x09, 0x47, 0xc7, 0x41, 0xe4, 0x35, 0xbb,
-	0xd6, 0xae, 0xc3, 0xb4, 0x64, 0x70, 0x3e, 0xf5, 0xa0, 0xc4, 0xf9, 0xb4, 0x08, 0xb7, 0x35, 0x1f,
-	0xee, 0xe3, 0x78, 0x90, 0xf1, 0x68, 0xc4, 0x93, 0xd1, 0xb3, 0x40, 0xbc, 0xf0, 0xd6, 0x55, 0xb8,
-	0xf3, 0xa8, 0xdc, 0x7b, 0xc0, 0x53, 0xe1, 0xb5, 0xd1, 0x22, 0x7e, 0x93, 0xeb, 0xd0, 0x38, 0x08,
-	0xb2, 0x9e, 0x98, 0x64, 0x67, 0xde, 0x46, 0xd7, 0xda, 0x75, 0x59, 0x21, 0x93, 0x6d, 0xa8, 0x0d,
-	0x7c, 0x1e, 0x0a, 0x6f, 0x13, 0x37, 0x28, 0x81, 0x50, 0x58, 0x7f, 0x10, 0x27, 0x22, 0x78, 0x1e,
-	0x61, 0x11, 0xbc, 0x0e, 0x06, 0x35, 0x87, 0x91, 0xff, 0x81, 0x23, 0x43, 0xda, 0xea, 0x5a, 0xbb,
-	0xad, 0xbb, 0xad, 0xbd, 0xc9, 0x70, 0xaf, 0x27, 0xfc, 0x60, 0xcc, 0x43, 0x26, 0x71, 0x54, 0xf3,
-	0xa9, 0x47, 0x96, 0xa9, 0xf9, 0x54, 0x72, 0x92, 0x29, 0x7a, 0x1a, 0x05, 0x99, 0x77, 0x05, 0xad,
-	0x17, 0x32, 0xa5, 0xb0, 0xd1, 0x1f, 0x4f, 0xe2, 0x24, 0x63, 0x22, 0x9d, 0xc4, 0x51, 0x2a, 0x48,
-	0x07, 0x9c, 0xa3, 0x24, 0xf1, 0x2c, 0x5c, 0x28, 0x3f, 0xe9, 0xd7, 0xd0, 0x39, 0x08, 0x63, 0xff,
-	0xbc, 0xc7, 0x33, 0xce, 0xc4, 0x57, 0xb9, 0x48, 0x33, 0x19, 0x8b, 0xa2, 0xab, 0xd6, 0x29, 0x41,
-	0xa2, 0x58, 0x7f, 0xcf, 0x56, 0x28, 0x0a, 0x32, 0x4f, 0x98, 0x45, 0x55, 0x2e, 0xfc, 0xc6, 0x5c,
-	0x9c, 0xf1, 0x64, 0x84, 0x35, 0x76, 0x99, 0x12, 0x24, 0x8a, 0x9e, 0xb0, 0x2f, 0x5c, 0xa6, 0x04,
-	0xda, 0x87, 0xad, 0x8a, 0x7f, 0x4d, 0xf3, 0x2a, 0xac, 0xb1, 0xf8, 0x45, 0xbf, 0x97, 0x7a, 0x56,
-	0xd7, 0xd9, 0x75, 0x99, 0x96, 0xb0, 0x81, 0xe2, 0x30, 0x1f, 0x47, 0x52, 0x65, 0xa3, 0xaa, 0x04,
-	0xe8, 0x35, 0xa8, 0x61, 0x37, 0xc9, 0x28, 0xcb, 0xbd, 0xf2, 0x93, 0x7e, 0x63, 0x41, 0xf3, 0x98,
-	0x4f, 0x91, 0x48, 0x4a, 0xee, 0x41, 0xc3, 0xd4, 0x1a, 0x17, 0xb5, 0xee, 0xde, 0x90, 0x79, 0x2d,
-	0x16, 0xec, 0x19, 0xed, 0x51, 0x94, 0x25, 0x33, 0x56, 0x2c, 0xbe, 0xfe, 0x09, 0xb4, 0xe7, 0x54,
-	0xd2, 0xd3, 0xb9, 0x98, 0x99, 0x7c, 0x9e, 0x8b, 0x99, 0x8c, 0xf2, 0x82, 0x87, 0xb9, 0xc0, 0x2c,
-	0xb9, 0x4c, 0x09, 0x1f, 0xdb, 0x1f, 0x59, 0xf4, 0x19, 0x90, 0xc3, 0x44, 0xf0, 0x4c, 0xa0, 0x93,
-	0x63, 0x91, 0xa6, 0xfc, 0xb9, 0x58, 0x95, 0x6b, 0xa7, 0x9a, 0xeb, 0x22, 0xaf, 0x76, 0x25, 0xaf,
-	0xf4, 0x0e, 0x90, 0x9e, 0x08, 0x45, 0x26, 0xf4, 0x39, 0x7f, 0x8b, 0x5d, 0x7a, 0x6e, 0x38, 0xac,
-	0x5e, 0x4b, 0x6e, 0x81, 0x2b, 0x87, 0x06, 0x3a, 0x6b, 0xdd, 0x6d, 0xcb, 0x0c, 0x15, 0x93, 0x84,
-	0xa1, 0x0a, 0xeb, 0x81, 0xe6, 0x46, 0xfb, 0x19, 0x52, 0x75, 0x58, 0x09, 0xd0, 0xef, 0x2c, 0xe3,
-	0x0d, 0xe9, 0xff, 0xcd, 0x88, 0xe7, 0xba, 0xeb, 0x1d, 0xcd, 0xc1, 0x41, 0x0e, 0x1d, 0xc9, 0xa1,
-	0x3a, 0x83, 0x96, 0xd1, 0x70, 0x17, 0x69, 0xdc, 0x37, 0xf9, 0xb9, 0x2c, 0x0b, 0xea, 0xc3, 0x0d,
-	0x65, 0x61, 0xff, 0x82, 0x07, 0x21, 0x1f, 0x86, 0xff, 0xa8, 0x84, 0x73, 0x01, 0x79, 0x50, 0xc7,
-	0xbd, 0xfd, 0x9e, 0x3e, 0x06, 0x46, 0xa4, 0x39, 0x94, 0x27, 0xea, 0x31, 0x1f, 0x0b, 0x6d, 0x0d,
-	0xbf, 0x8b, 0x3c, 0xd8, 0x6f, 0xcd, 0xc3, 0x36, 0xd4, 0xe4, 0xf9, 0x93, 0xf3, 0xdd, 0x91, 0x2e,
-	0x51, 0x58, 0x91, 0x9d, 0xf7, 0x61, 0x6d, 0xe0, 0x9f, 0x89, 0x31, 0x27, 0xff, 0x87, 0x3a, 0x32,
-	0x17, 0xa9, 0x3e, 0x14, 0xcd, 0xa2, 0xe4, 0xcc, 0x68, 0xe8, 0xf7, 0x96, 0x0e, 0x76, 0x29, 0xcd,
-	0x39, 0x57, 0xf6, 0x82, 0x2b, 0x72, 0x1b, 0xea, 0x9a, 0x2f, 0x4e, 0x8b, 0xd7, 0x7a, 0xca, 0x68,
-	0xc9, 0x2d, 0x58, 0xc3, 0xe8, 0x52, 0xcf, 0x2d, 0x89, 0x20, 0xc2, 0xb4, 0x82, 0x1e, 0x81, 0xf3,
-	0x94, 0xf5, 0xe5, 0xa0, 0x40, 0xf6, 0x86, 0x86, 0x96, 0x24, 0xb9, 0x4f, 0xe3, 0x34, 0xd3, 0xb9,
-	0xc7, 0x6f, 0x89, 0x9d, 0xc4, 0x89, 0xea, 0xd3, 0x36, 0xc3, 0x6f, 0xfa, 0xa3, 0x05, 0xee, 0xe3,
-	0x78, 0x24, 0xc8, 0x06, 0xd8, 0xfd, 0x9e, 0x36, 0x62, 0xf7, 0x7b, 0xe4, 0x1a, 0xda, 0xd7, 0xf9,
-	0xae, 0x4b, 0xff, 0x4f, 0x59, 0x9f, 0xa1, 0xcf, 0x9b, 0xd0, 0xec, 0xa7, 0x27, 0x49, 0x30, 0xe6,
-	0xc9, 0x4c, 0xdf, 0xa4, 0x25, 0x80, 0x67, 0x34, 0xe3, 0x99, 0xba, 0xdf, 0x9a, 0x4c, 0x09, 0xe4,
-	0x16, 0xd4, 0x1f, 0xb2, 0x93, 0x43, 0x69, 0xb2, 0x36, 0x6f, 0xd2, 0xe0, 0xf4, 0x3e, 0x74, 0x24,
-	0x13, 0x5c, 0x6f, 0x3a, 0xeb, 0x2a, 0xac, 0x49, 0xac, 0x60, 0xa6, 0xa5, 0xd2, 0x89, 0x5d, 0x71,
-	0x42, 0x1f, 0x28, 0x0b, 0x47, 0x17, 0x22, 0xca, 0x2a, 0xbd, 0x89, 0x32, 0x1a, 0x68, 0x33, 0x25,
-	0x90, 0x9b, 0x2a, 0x6a, 0x1d, 0x5e, 0x43, 0x72, 0x91, 0x32, 0x43, 0x94, 0xce, 0x00, 0x0c, 0x93,
-	0x3c, 0x2d, 0xd6, 0x5a, 0xcb, 0xd6, 0x12, 0x6a, 0xda, 0x47, 0x1f, 0x51, 0x90, 0x7a, 0x85, 0x30,
-	0xd3, 0x58, 0xef, 0x95, 0x8d, 0xa5, 0xea, 0xb9, 0x59, 0xd4, 0x5d, 0xf9, 0x28, 0xdb, 0xeb, 0x0c,
-	0x5a, 0x15, 0x7c, 0x69, 0x8f, 0xdd, 0x2e, 0x9a, 0xc3, 0x2e, 0x8d, 0x21, 0xa2, 0x8d, 0x69, 0xf5,
-	0x8a, 0xe1, 0x14, 0x40, 0xab, 0xb2, 0x69, 0xa9, 0xa7, 0x5d, 0xd8, 0x9c, 0x3f, 0xf0, 0xe6, 0xce,
-	0x59, 0x84, 0x57, 0xb8, 0xfa, 0xc1, 0x82, 0xf6, 0x61, 0x98, 0xa7, 0x99, 0x48, 0x8a, 0x9c, 0x36,
-	0x35, 0x50, 0x94, 0xb6, 0x04, 0x96, 0x57, 0x97, 0xec, 0x40, 0x4d, 0x66, 0x5c, 0x1d, 0xee, 0x6a,
-	0x21, 0x14, 0x5c, 0xa9, 0x84, 0xfb, 0xa6, 0x4a, 0xd0, 0x67, 0xd0, 0x38, 0x18, 0xf4, 0x1f, 0x26,
-	0x71, 0x3e, 0x59, 0x1a, 0xb1, 0x79, 0xd2, 0xd9, 0x95, 0x27, 0x5d, 0x47, 0x3d, 0x4f, 0x54, 0x54,
-	0xf8, 0x22, 0xe9, 0xa8, 0x17, 0x89, 0xab, 0x11, 0x3e, 0xa5, 0x03, 0xd8, 0x52, 0xe1, 0xca, 0x89,
-	0x73, 0x99, 0xb1, 0x68, 0x5e, 0x11, 0x4e, 0xf9, 0x8a, 0x90, 0x46, 0xd5, 0xd4, 0xfd, 0x37, 0x8d,
-	0xfe, 0x66, 0xc3, 0x16, 0x13, 0x69, 0xf0, 0x52, 0xf4, 0xa3, 0x34, 0x4b, 0x72, 0x5f, 0x4e, 0x1c,
-	0xb9, 0xff, 0xb3, 0x78, 0xa8, 0x6b, 0xe1, 0x30, 0x25, 0xbc, 0xfd, 0x94, 0x10, 0x0a, 0xf5, 0xea,
-	0x10, 0xa8, 0x2e, 0x30, 0x0a, 0x72, 0x07, 0xea, 0x83, 0x38, 0x4f, 0xfc, 0xa2, 0xf3, 0x71, 0x72,
-	0x2b, 0xff, 0x4a, 0xc1, 0xcc, 0x02, 0xf2, 0x08, 0xc8, 0x69, 0xc2, 0xa3, 0x34, 0xe4, 0x92, 0x92,
-	0xd9, 0xd6, 0x28, 0x9f, 0x27, 0x15, 0xed, 0x9c, 0x85, 0x25, 0xdb, 0xc8, 0x5e, 0xf5, 0x08, 0x7b,
-	0x75, 0xe4, 0xb7, 0x61, 0xf8, 0xe9, 0x73, 0x52, 0x3d, 0xe4, 0xf7, 0x16, 0x3a, 0xd4, 0x5b, 0xc3,
-	0x2d, 0x5b, 0x72, 0xcb, 0x9c, 0x82, 0xcd, 0xaf, 0xa3, 0xdf, 0x5a, 0xb0, 0x5e, 0x65, 0xb3, 0x62,
-	0x5c, 0x14, 0xe5, 0xb3, 0x57, 0xbf, 0x76, 0x4c, 0xf9, 0xdc, 0x65, 0x2f, 0xcb, 0x5a, 0xf5, 0x05,
-	0x14, 0xc3, 0x7f, 0xdf, 0x90, 0x9c, 0x4b, 0xd1, 0xe9, 0x42, 0xeb, 0x84, 0x27, 0x59, 0x20, 0x8d,
-	0xe9, 0x7b, 0xba, 0xc6, 0xaa, 0x10, 0x15, 0x70, 0xed, 0xb5, 0x26, 0x3a, 0x8c, 0xc7, 0x13, 0xd9,
-	0xad, 0x97, 0x6a, 0x26, 0x39, 0xa6, 0x93, 0x24, 0x4e, 0x4c, 0x06, 0x50, 0xa0, 0x07, 0xd0, 0x38,
-	0x8d, 0x27, 0x71, 0x18, 0x3f, 0x9f, 0xad, 0x18, 0x19, 0x1e, 0xd4, 0xd5, 0xd5, 0xa0, 0x46, 0x54,
-	0x93, 0x19, 0x91, 0x5e, 0x91, 0xfd, 0xee, 0xf3, 0xd0, 0xcf, 0x43, 0x9e, 0x09, 0x7c, 0x1f, 0x23,
-	0xf8, 0x79, 0xcc, 0x47, 0x6a, 0x2a, 0xe8, 0xa3, 0x45, 0xbf, 0xd4, 0x0d, 0xc8, 0x31, 0x9c, 0xca,
-	0x15, 0xb4, 0x8f, 0x80, 0xb9, 0x82, 0x94, 0x44, 0x3e, 0x80, 0x56, 0x65, 0xb5, 0x0e, 0x6b, 0xb3,
-	0xe8, 0x53, 0x05, 0xb3, 0xea, 0x1a, 0xfa, 0x8b, 0x35, 0xb7, 0xe7, 0xb5, 0x3b, 0x57, 0xbb, 0xba,
-	0x50, 0x49, 0x6a, 0x30, 0x2d, 0xc9, 0xd0, 0x8f, 0xa6, 0x7e, 0x98, 0xa7, 0x52, 0xa5, 0x2f, 0xdc,
-	0x02, 0x90, 0xa1, 0xcb, 0x1f, 0x9e, 0x38, 0x37, 0x8f, 0x1b, 0x23, 0xca, 0x5f, 0xa3, 0x9e, 0xe0,
-	0xa3, 0x30, 0x88, 0x04, 0xf6, 0x8b, 0xc3, 0x0a, 0x99, 0xdc, 0x51, 0x33, 0xd6, 0x34, 0xfa, 0xf6,
-	0x02, 0x71, 0xd4, 0xa9, 0xc9, 0x9b, 0x52, 0x02, 0x9d, 0x45, 0x15, 0xdd, 0x06, 0xa2, 0x3a, 0x60,
-	0x7f, 0x18, 0x27, 0xe6, 0xb6, 0xa5, 0x87, 0x66, 0xb8, 0xc8, 0xec, 0xaf, 0xba, 0xc4, 0xcb, 0xcc,
-	0xda, 0xd5, 0xcc, 0x1e, 0x74, 0x7e, 0x7d, 0xb5, 0x63, 0xfd, 0xfe, 0x6a, 0xc7, 0xfa, 0xe3, 0xd5,
-	0x8e, 0xf5, 0xd3, 0x9f, 0x3b, 0xff, 0x19, 0xae, 0xe1, 0xaf, 0xfc, 0x87, 0x7f, 0x05, 0x00, 0x00,
-	0xff, 0xff, 0x6d, 0xf8, 0xe6, 0x6b, 0xed, 0x0f, 0x00, 0x00,
+	// 1639 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0xdf, 0x6e, 0x1b, 0x45,
+	0x17, 0xff, 0x76, 0xd7, 0x8e, 0xed, 0xe3, 0x38, 0x71, 0xa6, 0xf9, 0xfa, 0x6d, 0xd2, 0x7e, 0x91,
+	0x33, 0xa0, 0xd6, 0x44, 0x22, 0x88, 0xf4, 0xa2, 0x08, 0x6e, 0x9a, 0xd8, 0x69, 0x31, 0x25, 0x6d,
+	0x3a, 0x49, 0x73, 0x09, 0x9a, 0xd8, 0xa3, 0x64, 0x95, 0xf5, 0xae, 0xd9, 0x5d, 0xa7, 0x76, 0x2f,
+	0x90, 0x40, 0x20, 0xb8, 0xe1, 0x9e, 0x2b, 0x9e, 0x81, 0x1b, 0xde, 0x81, 0x1b, 0x24, 0x1e, 0x01,
+	0x95, 0x17, 0x41, 0x73, 0x66, 0x66, 0x77, 0xed, 0x3a, 0x35, 0x44, 0xdc, 0xed, 0xf9, 0x9d, 0x99,
+	0xf3, 0x7f, 0xce, 0x9c, 0x59, 0xa8, 0x0d, 0x22, 0xef, 0x92, 0x27, 0x62, 0x7b, 0x10, 0x85, 0x49,
+	0x48, 0xec, 0xc1, 0xe9, 0xfa, 0xe2, 0x60, 0x78, 0xea, 0x7b, 0x5d, 0x85, 0xd0, 0x47, 0x50, 0xe9,
+	0x04, 0x3d, 0x31, 0x3a, 0x10, 0x09, 0x27, 0x04, 0x0a, 0x8f, 0xc5, 0x38, 0x76, 0x9d, 0x86, 0xd5,
+	0x2c, 0x33, 0xfc, 0x26, 0x77, 0x60, 0xe9, 0x38, 0xe2, 0xdd, 0x8b, 0xfd, 0x91, 0x17, 0x27, 0x22,
+	0xe8, 0x0a, 0xb7, 0x80, 0xdc, 0x29, 0x94, 0xfe, 0xec, 0xc0, 0xe2, 0x43, 0x4f, 0xf8, 0xbd, 0xa7,
+	0x83, 0xc4, 0x0b, 0x83, 0x58, 0x0a, 0x3b, 0x1e, 0x0f, 0x84, 0x5b, 0x6e, 0x58, 0xcd, 0x0a, 0xc3,
+	0x6f, 0x72, 0x1b, 0x2a, 0x2d, 0xde, 0x3d, 0x17, 0xc8, 0x70, 0x90, 0x91, 0x01, 0x29, 0xf7, 0xc8,
+	0x7b, 0xa9, 0xb4, 0xd4, 0x58, 0x06, 0x90, 0x06, 0x54, 0x8f, 0xbd, 0xbe, 0x78, 0x36, 0xe4, 0x41,
+	0x32, 0xec, 0xbb, 0x45, 0xdc, 0x9d, 0x87, 0xc8, 0x4d, 0x58, 0x78, 0xea, 0xf7, 0x0e, 0xbc, 0xc0,
+	0xad, 0x34, 0xac, 0xa6, 0xc3, 0x34, 0x65, 0x70, 0x3e, 0x72, 0x21, 0xc3, 0xf9, 0x28, 0x75, 0xb7,
+	0x3a, 0xe9, 0xee, 0x93, 0xf0, 0x28, 0xe1, 0x41, 0x8f, 0x47, 0xbd, 0x13, 0x4f, 0xbc, 0x70, 0x17,
+	0x95, 0xbb, 0x93, 0xa8, 0xdc, 0xbb, 0xc7, 0x63, 0xe1, 0xd6, 0x50, 0x22, 0x7e, 0x93, 0x75, 0x28,
+	0xef, 0x79, 0x49, 0x5b, 0x0c, 0x92, 0x73, 0x77, 0xa9, 0x61, 0x35, 0x0b, 0x2c, 0xa5, 0xc9, 0x2a,
+	0x14, 0x8f, 0xba, 0xdc, 0x17, 0xee, 0x32, 0x6e, 0x50, 0x04, 0xa1, 0xb0, 0xf8, 0x30, 0x8c, 0x84,
+	0x77, 0x16, 0x60, 0x12, 0xdc, 0x3a, 0x3a, 0x35, 0x81, 0x91, 0xff, 0x83, 0x23, 0x5d, 0x5a, 0x69,
+	0x58, 0xcd, 0xea, 0x4e, 0x75, 0x7b, 0x70, 0xba, 0xdd, 0x16, 0x5d, 0xaf, 0xcf, 0x7d, 0x26, 0x71,
+	0x64, 0xf3, 0x91, 0x4b, 0x66, 0xb1, 0xf9, 0x48, 0xda, 0x24, 0x43, 0xf4, 0x3c, 0xf0, 0x12, 0xf7,
+	0x06, 0x4a, 0x4f, 0x69, 0x4a, 0x61, 0xa9, 0xd3, 0x1f, 0x84, 0x51, 0xc2, 0x44, 0x3c, 0x08, 0x83,
+	0x58, 0x90, 0x3a, 0x38, 0xfb, 0x51, 0xe4, 0x5a, 0xb8, 0x50, 0x7e, 0xd2, 0x2f, 0xa1, 0xbe, 0xe7,
+	0x87, 0xdd, 0x8b, 0x36, 0x4f, 0x38, 0x13, 0x5f, 0x0c, 0x45, 0x9c, 0x48, 0x5f, 0x94, 0xb9, 0x6a,
+	0x9d, 0x22, 0x24, 0x8a, 0xf9, 0x77, 0x6d, 0x85, 0x22, 0x21, 0xe3, 0x84, 0x51, 0x54, 0xe9, 0xc2,
+	0x6f, 0x8c, 0xc5, 0x39, 0x8f, 0x7a, 0x98, 0xe3, 0x02, 0x53, 0x84, 0x44, 0x51, 0x13, 0xd6, 0x45,
+	0x81, 0x29, 0x82, 0x76, 0x60, 0x25, 0xa7, 0x5f, 0x9b, 0x79, 0x13, 0x16, 0x58, 0xf8, 0xa2, 0xd3,
+	0x8e, 0x5d, 0xab, 0xe1, 0x34, 0x0b, 0x4c, 0x53, 0x58, 0x40, 0xa1, 0x3f, 0xec, 0x07, 0x92, 0x65,
+	0x23, 0x2b, 0x03, 0xe8, 0x1a, 0x14, 0xb1, 0x9a, 0xa4, 0x97, 0xd9, 0x5e, 0xf9, 0x49, 0xbf, 0xb2,
+	0xa0, 0x72, 0xc0, 0x47, 0x68, 0x48, 0x4c, 0xee, 0x43, 0xd9, 0xe4, 0x1a, 0x17, 0x55, 0x77, 0x6e,
+	0xc9, 0xb8, 0xa6, 0x0b, 0xb6, 0x0d, 0x77, 0x3f, 0x48, 0xa2, 0x31, 0x4b, 0x17, 0xaf, 0x7f, 0x04,
+	0xb5, 0x09, 0x96, 0xd4, 0x74, 0x21, 0xc6, 0x26, 0x9e, 0x17, 0x62, 0x2c, 0xbd, 0xbc, 0xe4, 0xfe,
+	0x50, 0x60, 0x94, 0x0a, 0x4c, 0x11, 0x1f, 0xda, 0x1f, 0x58, 0xf4, 0x04, 0x48, 0x2b, 0x12, 0x3c,
+	0x11, 0xa8, 0xe4, 0x40, 0xc4, 0x31, 0x3f, 0x13, 0xf3, 0x62, 0xed, 0xe4, 0x63, 0x9d, 0xc6, 0xd5,
+	0xce, 0xc5, 0x95, 0x6e, 0x01, 0x69, 0x0b, 0x5f, 0x24, 0x42, 0x9f, 0xf3, 0x37, 0xc8, 0xa5, 0x17,
+	0xc6, 0x86, 0xf9, 0x6b, 0xc9, 0x26, 0x14, 0x64, 0xd3, 0x40, 0x65, 0xd5, 0x9d, 0x9a, 0x8c, 0x50,
+	0xda, 0x49, 0x18, 0xb2, 0x30, 0x1f, 0x28, 0xae, 0xb7, 0x9b, 0xa0, 0xa9, 0x0e, 0xcb, 0x00, 0xfa,
+	0x8d, 0x65, 0xb4, 0xa1, 0xf9, 0x7f, 0xd3, 0xe3, 0x89, 0xea, 0x7a, 0x5b, 0xdb, 0xe0, 0xa0, 0x0d,
+	0x75, 0x69, 0x43, 0xbe, 0x07, 0xcd, 0x32, 0xa3, 0x30, 0x6d, 0xc6, 0x03, 0x13, 0x9f, 0xeb, 0x5a,
+	0x41, 0xbb, 0x70, 0x4b, 0x49, 0xd8, 0xbd, 0xe4, 0x9e, 0xcf, 0x4f, 0xfd, 0x7f, 0x94, 0xc2, 0x09,
+	0x87, 0x5c, 0x28, 0xe1, 0xde, 0x4e, 0x5b, 0x1f, 0x03, 0x43, 0xd2, 0x21, 0x64, 0x27, 0xea, 0x09,
+	0xef, 0x0b, 0x2d, 0x0d, 0xbf, 0xd3, 0x38, 0xd8, 0x6f, 0x8c, 0xc3, 0x2a, 0x14, 0xe5, 0xf9, 0x93,
+	0xfd, 0xdd, 0x91, 0x2a, 0x91, 0x98, 0x13, 0x9d, 0x77, 0x61, 0xe1, 0xa8, 0x7b, 0x2e, 0xfa, 0x9c,
+	0xbc, 0x05, 0x25, 0xb4, 0x5c, 0xc4, 0xfa, 0x50, 0x54, 0xd2, 0x94, 0x33, 0xc3, 0xa1, 0xdf, 0x5a,
+	0xda, 0xd9, 0x99, 0x66, 0x4e, 0xa8, 0xb2, 0xa7, 0x54, 0x91, 0xbb, 0x50, 0xd2, 0xf6, 0x62, 0xb7,
+	0x78, 0xad, 0xa6, 0x0c, 0x97, 0x6c, 0xc2, 0x02, 0x7a, 0x17, 0xbb, 0x85, 0xcc, 0x10, 0x44, 0x98,
+	0x66, 0xd0, 0x7d, 0x70, 0x9e, 0xb3, 0x8e, 0x6c, 0x14, 0x68, 0xbd, 0x31, 0x43, 0x53, 0xd2, 0xb8,
+	0x8f, 0xc3, 0x38, 0xd1, 0xb1, 0xc7, 0x6f, 0x89, 0x1d, 0x86, 0x91, 0xaa, 0xd3, 0x1a, 0xc3, 0x6f,
+	0xfa, 0xbd, 0x05, 0x85, 0x27, 0x61, 0x4f, 0x90, 0x25, 0xb0, 0x3b, 0x6d, 0x2d, 0xc4, 0xee, 0xb4,
+	0xc9, 0x1a, 0xca, 0xd7, 0xf1, 0x2e, 0x49, 0xfd, 0xcf, 0x59, 0x87, 0xa1, 0xce, 0xdb, 0x50, 0xe9,
+	0xc4, 0x87, 0x91, 0xd7, 0xe7, 0xd1, 0x58, 0xdf, 0xa4, 0x19, 0x80, 0x67, 0x34, 0xe1, 0x89, 0xba,
+	0xdf, 0x2a, 0x4c, 0x11, 0x64, 0x13, 0x4a, 0x8f, 0xd8, 0x61, 0x4b, 0x8a, 0x2c, 0x4e, 0x8a, 0x34,
+	0x38, 0x7d, 0x00, 0x75, 0x69, 0x09, 0xae, 0x37, 0x95, 0x75, 0x13, 0x16, 0x24, 0x96, 0x5a, 0xa6,
+	0xa9, 0x4c, 0x89, 0x9d, 0x53, 0x42, 0x1f, 0x2a, 0x09, 0xfb, 0x97, 0x22, 0x48, 0x72, 0xb5, 0x89,
+	0x34, 0x0a, 0xa8, 0x31, 0x45, 0x90, 0xdb, 0xca, 0x6b, 0xed, 0x5e, 0x59, 0xda, 0x22, 0x69, 0x86,
+	0x28, 0x1d, 0x03, 0x18, 0x4b, 0x86, 0x71, 0xba, 0xd6, 0x9a, 0xb5, 0x96, 0x50, 0x53, 0x3e, 0xfa,
+	0x88, 0x82, 0xe4, 0x2b, 0x84, 0x99, 0xc2, 0x7a, 0x27, 0x2b, 0x2c, 0x95, 0xcf, 0xe5, 0x34, 0xef,
+	0x4a, 0x47, 0x56, 0x5e, 0xe7, 0x50, 0xcd, 0xe1, 0x33, 0x6b, 0xec, 0x6e, 0x5a, 0x1c, 0x76, 0x26,
+	0x0c, 0x11, 0x2d, 0x4c, 0xb3, 0xe7, 0x34, 0x27, 0x0f, 0xaa, 0xb9, 0x4d, 0x33, 0x35, 0x35, 0x61,
+	0x79, 0xf2, 0xc0, 0x9b, 0x3b, 0x67, 0x1a, 0x9e, 0xa3, 0xea, 0x3b, 0x0b, 0x6a, 0x2d, 0x7f, 0x18,
+	0x27, 0x22, 0x4a, 0x63, 0x5a, 0xd1, 0x40, 0x9a, 0xda, 0x0c, 0x98, 0x9d, 0x5d, 0xb2, 0x01, 0x45,
+	0x19, 0x71, 0x75, 0xb8, 0xf3, 0x89, 0x50, 0x70, 0x2e, 0x13, 0x85, 0xab, 0x32, 0x41, 0x4f, 0xa0,
+	0xbc, 0x77, 0xd4, 0x79, 0x14, 0x85, 0xc3, 0xc1, 0x4c, 0x8f, 0xcd, 0x48, 0x67, 0xe7, 0x46, 0xba,
+	0xba, 0x1a, 0x4f, 0x94, 0x57, 0x38, 0x91, 0xd4, 0xd5, 0x44, 0x52, 0xd0, 0x08, 0x1f, 0xd1, 0x23,
+	0x58, 0x51, 0xee, 0xca, 0x8e, 0x73, 0x9d, 0xb6, 0x68, 0xa6, 0x08, 0x27, 0x9b, 0x22, 0xa4, 0x50,
+	0xd5, 0x75, 0xff, 0x4d, 0xa1, 0xbf, 0xd9, 0xb0, 0xc2, 0x44, 0xec, 0xbd, 0x14, 0x9d, 0x20, 0x4e,
+	0xa2, 0x61, 0x57, 0x76, 0x1c, 0xb9, 0xff, 0x93, 0xf0, 0x54, 0xe7, 0xc2, 0x61, 0x8a, 0x78, 0xf3,
+	0x29, 0x21, 0x14, 0x4a, 0xf9, 0x26, 0x90, 0x5f, 0x60, 0x18, 0x64, 0x0b, 0x4a, 0x47, 0xe1, 0x30,
+	0xea, 0xa6, 0x95, 0x8f, 0x9d, 0x5b, 0xe9, 0x57, 0x0c, 0x66, 0x16, 0x90, 0xc7, 0x40, 0x8e, 0x23,
+	0x1e, 0xc4, 0x3e, 0x97, 0x26, 0x99, 0x6d, 0xe5, 0x6c, 0x3c, 0xc9, 0x71, 0x27, 0x24, 0xcc, 0xd8,
+	0x46, 0xb6, 0xf3, 0x47, 0xd8, 0x2d, 0xa1, 0x7d, 0x4b, 0xc6, 0x3e, 0x7d, 0x4e, 0xf2, 0x87, 0xfc,
+	0xfe, 0x54, 0x85, 0xba, 0x0b, 0xb8, 0x65, 0x45, 0x6e, 0x99, 0x60, 0xb0, 0xc9, 0x75, 0xf4, 0x6b,
+	0x0b, 0x16, 0xf3, 0xd6, 0xcc, 0x69, 0x17, 0x69, 0xfa, 0xec, 0xf9, 0xd3, 0x8e, 0x49, 0x5f, 0x61,
+	0xd6, 0x64, 0x59, 0xcc, 0x4f, 0x40, 0x21, 0xfc, 0xef, 0x8a, 0xe0, 0x5c, 0xcb, 0x9c, 0x06, 0x54,
+	0x0f, 0x79, 0x94, 0x78, 0x52, 0x98, 0xbe, 0xa7, 0x8b, 0x2c, 0x0f, 0x51, 0x01, 0x6b, 0xaf, 0x15,
+	0x51, 0x2b, 0xec, 0x0f, 0x64, 0xb5, 0x5e, 0xab, 0x98, 0x64, 0x9b, 0x8e, 0xa2, 0x30, 0x32, 0x11,
+	0x40, 0x82, 0xee, 0x41, 0xf9, 0x38, 0x1c, 0x84, 0x7e, 0x78, 0x36, 0x9e, 0xd3, 0x32, 0x5c, 0x28,
+	0xa9, 0xab, 0x41, 0xb5, 0xa8, 0x0a, 0x33, 0x24, 0xbd, 0x21, 0xeb, 0xbd, 0xcb, 0xfd, 0xee, 0xd0,
+	0xe7, 0x89, 0xc0, 0xf9, 0x18, 0xc1, 0x4f, 0x43, 0xde, 0x53, 0x5d, 0x41, 0x1f, 0x2d, 0xfa, 0xb9,
+	0x2e, 0x40, 0x8e, 0xee, 0xe4, 0xae, 0xa0, 0x5d, 0x04, 0xcc, 0x15, 0xa4, 0x28, 0xf2, 0x3e, 0x54,
+	0x73, 0xab, 0xb5, 0x5b, 0xcb, 0x69, 0x9d, 0x2a, 0x98, 0xe5, 0xd7, 0xd0, 0x5f, 0xac, 0x89, 0x3d,
+	0xaf, 0xdd, 0xb9, 0x5a, 0xd5, 0xa5, 0x0a, 0x52, 0x99, 0x69, 0x4a, 0xba, 0xbe, 0x3f, 0xea, 0xfa,
+	0xc3, 0x58, 0xb2, 0xf4, 0x85, 0x9b, 0x02, 0xd2, 0x75, 0xf9, 0xe0, 0x09, 0x87, 0x66, 0xb8, 0x31,
+	0xa4, 0x7c, 0x1a, 0xb5, 0x05, 0xef, 0xf9, 0x5e, 0x20, 0xb0, 0x5e, 0x1c, 0x96, 0xd2, 0x64, 0x4b,
+	0xf5, 0x58, 0x53, 0xe8, 0xab, 0x53, 0x86, 0x23, 0x4f, 0x75, 0xde, 0x98, 0x12, 0xa8, 0x4f, 0xb3,
+	0xe8, 0x2a, 0x10, 0x55, 0x01, 0xbb, 0xa7, 0x61, 0x64, 0x6e, 0x5b, 0xda, 0x32, 0xcd, 0x45, 0x46,
+	0x7f, 0xde, 0x25, 0x9e, 0x45, 0xd6, 0xce, 0x47, 0x96, 0x7e, 0x06, 0x4b, 0x7a, 0xb6, 0x13, 0x11,
+	0x16, 0xb4, 0x0c, 0x00, 0x13, 0xdd, 0x50, 0x8e, 0x89, 0xe6, 0x55, 0x93, 0x01, 0x52, 0xce, 0x89,
+	0x7c, 0x64, 0x98, 0xdb, 0x49, 0x53, 0x38, 0x1b, 0x79, 0x67, 0x81, 0xe8, 0xe1, 0x8d, 0xe1, 0x30,
+	0x4d, 0xd1, 0x1f, 0x6c, 0x58, 0x55, 0x43, 0x67, 0x70, 0x26, 0xe2, 0x24, 0x53, 0x23, 0x9f, 0xd1,
+	0x03, 0xec, 0xff, 0xda, 0x50, 0x45, 0xc9, 0x27, 0x73, 0xcb, 0x17, 0x3c, 0xca, 0x6c, 0x50, 0x8a,
+	0xa6, 0x50, 0x79, 0x6e, 0x10, 0xd1, 0xd7, 0xb3, 0x1a, 0x42, 0xf3, 0x10, 0xd9, 0x83, 0xb2, 0x76,
+	0xcd, 0x34, 0xc4, 0x3b, 0x78, 0x4b, 0xcd, 0xb0, 0xc6, 0xcc, 0xb7, 0xb1, 0x7e, 0x83, 0x19, 0x72,
+	0xfd, 0x29, 0xd4, 0x26, 0x58, 0x33, 0xde, 0x60, 0xcd, 0xfc, 0x1b, 0xac, 0xba, 0x43, 0x72, 0xe3,
+	0xb2, 0x96, 0x9e, 0x7f, 0x97, 0xb5, 0xe0, 0xbf, 0xb3, 0x0c, 0x88, 0xc9, 0x16, 0x38, 0xd2, 0x50,
+	0x35, 0x0c, 0xbb, 0x57, 0x19, 0xca, 0xe4, 0x22, 0xfa, 0x93, 0xa5, 0x83, 0x2a, 0x34, 0xdf, 0xbc,
+	0xa5, 0xef, 0xe5, 0x85, 0x6c, 0xa6, 0x42, 0xa6, 0x96, 0x6d, 0xa7, 0x8e, 0xca, 0xd5, 0xeb, 0xcf,
+	0xa0, 0x3c, 0xcb, 0xbd, 0x82, 0x72, 0xef, 0xbd, 0x49, 0xf7, 0xd6, 0xae, 0xb2, 0x2c, 0xce, 0x79,
+	0xb9, 0x57, 0xff, 0xf5, 0xd5, 0x86, 0xf5, 0xfb, 0xab, 0x0d, 0xeb, 0x8f, 0x57, 0x1b, 0xd6, 0x8f,
+	0x7f, 0x6e, 0xfc, 0xe7, 0x74, 0x01, 0x7f, 0x10, 0xdd, 0xfb, 0x2b, 0x00, 0x00, 0xff, 0xff, 0x38,
+	0x55, 0x86, 0x89, 0x43, 0x12, 0x00, 0x00,
 }
 
 func (m *IndexMeta) Marshal() (dAtA []byte, err error) {
@@ -4680,6 +4926,267 @@ func (m *ResizeNodeMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	return len(dAtA) - i, nil
 }
 
+func (m *FieldOperation) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *FieldOperation) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *FieldOperation) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if m.XXX_unrecognized != nil {
+		i -= len(m.XXX_unrecognized)
+		copy(dAtA[i:], m.XXX_unrecognized)
+	}
+	if len(m.Signed) > 0 {
+		dAtA31 := make([]byte, len(m.Signed)*10)
+		var j30 int
+		for _, num1 := range m.Signed {
+			num := uint64(num1)
+			for num >= 1<<7 {
+				dAtA31[j30] = uint8(uint64(num)&0x7f | 0x80)
+				num >>= 7
+				j30++
+			}
+			dAtA31[j30] = uint8(num)
+			j30++
+		}
+		i -= j30
+		copy(dAtA[i:], dAtA31[:j30])
+		i = encodeVarintPrivate(dAtA, i, uint64(j30))
+		i--
+		dAtA[i] = 0x1a
+	}
+	if len(m.Values) > 0 {
+		dAtA33 := make([]byte, len(m.Values)*10)
+		var j32 int
+		for _, num := range m.Values {
+			for num >= 1<<7 {
+				dAtA33[j32] = uint8(uint64(num)&0x7f | 0x80)
+				num >>= 7
+				j32++
+			}
+			dAtA33[j32] = uint8(num)
+			j32++
+		}
+		i -= j32
+		copy(dAtA[i:], dAtA33[:j32])
+		i = encodeVarintPrivate(dAtA, i, uint64(j32))
+		i--
+		dAtA[i] = 0x12
+	}
+	if len(m.RecordIDs) > 0 {
+		dAtA35 := make([]byte, len(m.RecordIDs)*10)
+		var j34 int
+		for _, num := range m.RecordIDs {
+			for num >= 1<<7 {
+				dAtA35[j34] = uint8(uint64(num)&0x7f | 0x80)
+				num >>= 7
+				j34++
+			}
+			dAtA35[j34] = uint8(num)
+			j34++
+		}
+		i -= j34
+		copy(dAtA[i:], dAtA35[:j34])
+		i = encodeVarintPrivate(dAtA, i, uint64(j34))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
+}
+
+func (m *ShardIngestOperation) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *ShardIngestOperation) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *ShardIngestOperation) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if m.XXX_unrecognized != nil {
+		i -= len(m.XXX_unrecognized)
+		copy(dAtA[i:], m.XXX_unrecognized)
+	}
+	if len(m.FieldOps) > 0 {
+		for k := range m.FieldOps {
+			v := m.FieldOps[k]
+			baseI := i
+			if v != nil {
+				{
+					size, err := v.MarshalToSizedBuffer(dAtA[:i])
+					if err != nil {
+						return 0, err
+					}
+					i -= size
+					i = encodeVarintPrivate(dAtA, i, uint64(size))
+				}
+				i--
+				dAtA[i] = 0x12
+			}
+			i -= len(k)
+			copy(dAtA[i:], k)
+			i = encodeVarintPrivate(dAtA, i, uint64(len(k)))
+			i--
+			dAtA[i] = 0xa
+			i = encodeVarintPrivate(dAtA, i, uint64(baseI-i))
+			i--
+			dAtA[i] = 0x22
+		}
+	}
+	if len(m.ClearFields) > 0 {
+		for iNdEx := len(m.ClearFields) - 1; iNdEx >= 0; iNdEx-- {
+			i -= len(m.ClearFields[iNdEx])
+			copy(dAtA[i:], m.ClearFields[iNdEx])
+			i = encodeVarintPrivate(dAtA, i, uint64(len(m.ClearFields[iNdEx])))
+			i--
+			dAtA[i] = 0x1a
+		}
+	}
+	if len(m.ClearRecordIDs) > 0 {
+		dAtA38 := make([]byte, len(m.ClearRecordIDs)*10)
+		var j37 int
+		for _, num := range m.ClearRecordIDs {
+			for num >= 1<<7 {
+				dAtA38[j37] = uint8(uint64(num)&0x7f | 0x80)
+				num >>= 7
+				j37++
+			}
+			dAtA38[j37] = uint8(num)
+			j37++
+		}
+		i -= j37
+		copy(dAtA[i:], dAtA38[:j37])
+		i = encodeVarintPrivate(dAtA, i, uint64(j37))
+		i--
+		dAtA[i] = 0x12
+	}
+	if len(m.OpType) > 0 {
+		i -= len(m.OpType)
+		copy(dAtA[i:], m.OpType)
+		i = encodeVarintPrivate(dAtA, i, uint64(len(m.OpType)))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
+}
+
+func (m *ShardIngestOperations) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *ShardIngestOperations) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *ShardIngestOperations) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if m.XXX_unrecognized != nil {
+		i -= len(m.XXX_unrecognized)
+		copy(dAtA[i:], m.XXX_unrecognized)
+	}
+	if len(m.Ops) > 0 {
+		for iNdEx := len(m.Ops) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Ops[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintPrivate(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0xa
+		}
+	}
+	return len(dAtA) - i, nil
+}
+
+func (m *ShardedIngestRequest) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *ShardedIngestRequest) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *ShardedIngestRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if m.XXX_unrecognized != nil {
+		i -= len(m.XXX_unrecognized)
+		copy(dAtA[i:], m.XXX_unrecognized)
+	}
+	if len(m.Ops) > 0 {
+		for k := range m.Ops {
+			v := m.Ops[k]
+			baseI := i
+			if v != nil {
+				{
+					size, err := v.MarshalToSizedBuffer(dAtA[:i])
+					if err != nil {
+						return 0, err
+					}
+					i -= size
+					i = encodeVarintPrivate(dAtA, i, uint64(size))
+				}
+				i--
+				dAtA[i] = 0x12
+			}
+			i = encodeVarintPrivate(dAtA, i, uint64(k))
+			i--
+			dAtA[i] = 0x8
+			i = encodeVarintPrivate(dAtA, i, uint64(baseI-i))
+			i--
+			dAtA[i] = 0xa
+		}
+	}
+	return len(dAtA) - i, nil
+}
+
 func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int {
 	offset -= sovPrivate(v)
 	base := offset
@@ -5628,6 +6135,124 @@ func (m *ResizeNodeMessage) Size() (n int) {
 	return n
 }
 
+func (m *FieldOperation) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	if len(m.RecordIDs) > 0 {
+		l = 0
+		for _, e := range m.RecordIDs {
+			l += sovPrivate(uint64(e))
+		}
+		n += 1 + sovPrivate(uint64(l)) + l
+	}
+	if len(m.Values) > 0 {
+		l = 0
+		for _, e := range m.Values {
+			l += sovPrivate(uint64(e))
+		}
+		n += 1 + sovPrivate(uint64(l)) + l
+	}
+	if len(m.Signed) > 0 {
+		l = 0
+		for _, e := range m.Signed {
+			l += sovPrivate(uint64(e))
+		}
+		n += 1 + sovPrivate(uint64(l)) + l
+	}
+	if m.XXX_unrecognized != nil {
+		n += len(m.XXX_unrecognized)
+	}
+	return n
+}
+
+func (m *ShardIngestOperation) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = len(m.OpType)
+	if l > 0 {
+		n += 1 + l + sovPrivate(uint64(l))
+	}
+	if len(m.ClearRecordIDs) > 0 {
+		l = 0
+		for _, e := range m.ClearRecordIDs {
+			l += sovPrivate(uint64(e))
+		}
+		n += 1 + sovPrivate(uint64(l)) + l
+	}
+	if len(m.ClearFields) > 0 {
+		for _, s := range m.ClearFields {
+			l = len(s)
+			n += 1 + l + sovPrivate(uint64(l))
+		}
+	}
+	if len(m.FieldOps) > 0 {
+		for k, v := range m.FieldOps {
+			_ = k
+			_ = v
+			l = 0
+			if v != nil {
+				l = v.Size()
+				l += 1 + sovPrivate(uint64(l))
+			}
+			mapEntrySize := 1 + len(k) + sovPrivate(uint64(len(k))) + l
+			n += mapEntrySize + 1 + sovPrivate(uint64(mapEntrySize))
+		}
+	}
+	if m.XXX_unrecognized != nil {
+		n += len(m.XXX_unrecognized)
+	}
+	return n
+}
+
+func (m *ShardIngestOperations) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	if len(m.Ops) > 0 {
+		for _, e := range m.Ops {
+			l = e.Size()
+			n += 1 + l + sovPrivate(uint64(l))
+		}
+	}
+	if m.XXX_unrecognized != nil {
+		n += len(m.XXX_unrecognized)
+	}
+	return n
+}
+
+func (m *ShardedIngestRequest) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	if len(m.Ops) > 0 {
+		for k, v := range m.Ops {
+			_ = k
+			_ = v
+			l = 0
+			if v != nil {
+				l = v.Size()
+				l += 1 + sovPrivate(uint64(l))
+			}
+			mapEntrySize := 1 + sovPrivate(uint64(k)) + l
+			n += mapEntrySize + 1 + sovPrivate(uint64(mapEntrySize))
+		}
+	}
+	if m.XXX_unrecognized != nil {
+		n += len(m.XXX_unrecognized)
+	}
+	return n
+}
+
 func sovPrivate(x uint64) (n int) {
 	return (math_bits.Len64(x|1) + 6) / 7
 }
@@ -5709,10 +6334,7 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -6149,10 +6771,7 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -6235,10 +6854,7 @@ func (m *ImportResponse) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -6423,10 +7039,7 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -6629,10 +7242,7 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -6759,10 +7369,7 @@ func (m *Cache) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -6909,7 +7516,7 @@ func (m *MaxShards) Unmarshal(dAtA []byte) error {
 					if err != nil {
 						return err
 					}
-					if skippy < 0 {
+					if (skippy < 0) || (iNdEx+skippy) < 0 {
 						return ErrInvalidLengthPrivate
 					}
 					if (iNdEx + skippy) > postIndex {
@@ -6926,10 +7533,7 @@ func (m *MaxShards) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -7063,10 +7667,7 @@ func (m *CreateShardMessage) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -7149,10 +7750,7 @@ func (m *DeleteIndexMessage) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -7290,10 +7888,7 @@ func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -7463,10 +8058,7 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -7581,10 +8173,7 @@ func (m *DeleteFieldMessage) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -7718,10 +8307,7 @@ func (m *DeleteAvailableShardMessage) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -7891,10 +8477,7 @@ func (m *Field) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -7979,10 +8562,7 @@ func (m *Schema) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -8154,10 +8734,7 @@ func (m *Index) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -8291,10 +8868,7 @@ func (m *URI) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -8501,10 +9075,7 @@ func (m *Node) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -8619,10 +9190,7 @@ func (m *NodeStateMessage) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -8728,10 +9296,7 @@ func (m *NodeEventMessage) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -8888,10 +9453,7 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -9027,10 +9589,7 @@ func (m *IndexStatus) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -9208,10 +9767,7 @@ func (m *FieldStatus) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -9396,10 +9952,7 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -9552,10 +10105,7 @@ func (m *BSIGroup) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -9702,10 +10252,7 @@ func (m *CreateViewMessage) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -9852,10 +10399,7 @@ func (m *DeleteViewMessage) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -10137,10 +10681,7 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -10342,10 +10883,7 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -10483,10 +11021,7 @@ func (m *TranslationResizeSource) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -10624,10 +11159,7 @@ func (m *ResizeInstructionComplete) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -10742,10 +11274,7 @@ func (m *Topology) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -10796,10 +11325,7 @@ func (m *RecalculateCaches) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -10850,10 +11376,7 @@ func (m *LoadSchemaMessage) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -10972,10 +11495,7 @@ func (m *TransactionMessage) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -11172,10 +11692,7 @@ func (m *Transaction) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -11226,10 +11743,7 @@ func (m *TransactionStats) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -11280,10 +11794,7 @@ func (m *ResizeAbortMessage) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPrivate
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
@@ -11398,10 +11909,857 @@ func (m *ResizeNodeMessage) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
-			if (iNdEx + skippy) < 0 {
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func (m *FieldOperation) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowPrivate
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: FieldOperation: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: FieldOperation: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType == 0 {
+				var v uint64
+				for shift := uint(0); ; shift += 7 {
+					if shift >= 64 {
+						return ErrIntOverflowPrivate
+					}
+					if iNdEx >= l {
+						return io.ErrUnexpectedEOF
+					}
+					b := dAtA[iNdEx]
+					iNdEx++
+					v |= uint64(b&0x7F) << shift
+					if b < 0x80 {
+						break
+					}
+				}
+				m.RecordIDs = append(m.RecordIDs, v)
+			} else if wireType == 2 {
+				var packedLen int
+				for shift := uint(0); ; shift += 7 {
+					if shift >= 64 {
+						return ErrIntOverflowPrivate
+					}
+					if iNdEx >= l {
+						return io.ErrUnexpectedEOF
+					}
+					b := dAtA[iNdEx]
+					iNdEx++
+					packedLen |= int(b&0x7F) << shift
+					if b < 0x80 {
+						break
+					}
+				}
+				if packedLen < 0 {
+					return ErrInvalidLengthPrivate
+				}
+				postIndex := iNdEx + packedLen
+				if postIndex < 0 {
+					return ErrInvalidLengthPrivate
+				}
+				if postIndex > l {
+					return io.ErrUnexpectedEOF
+				}
+				var elementCount int
+				var count int
+				for _, integer := range dAtA[iNdEx:postIndex] {
+					if integer < 128 {
+						count++
+					}
+				}
+				elementCount = count
+				if elementCount != 0 && len(m.RecordIDs) == 0 {
+					m.RecordIDs = make([]uint64, 0, elementCount)
+				}
+				for iNdEx < postIndex {
+					var v uint64
+					for shift := uint(0); ; shift += 7 {
+						if shift >= 64 {
+							return ErrIntOverflowPrivate
+						}
+						if iNdEx >= l {
+							return io.ErrUnexpectedEOF
+						}
+						b := dAtA[iNdEx]
+						iNdEx++
+						v |= uint64(b&0x7F) << shift
+						if b < 0x80 {
+							break
+						}
+					}
+					m.RecordIDs = append(m.RecordIDs, v)
+				}
+			} else {
+				return fmt.Errorf("proto: wrong wireType = %d for field RecordIDs", wireType)
+			}
+		case 2:
+			if wireType == 0 {
+				var v uint64
+				for shift := uint(0); ; shift += 7 {
+					if shift >= 64 {
+						return ErrIntOverflowPrivate
+					}
+					if iNdEx >= l {
+						return io.ErrUnexpectedEOF
+					}
+					b := dAtA[iNdEx]
+					iNdEx++
+					v |= uint64(b&0x7F) << shift
+					if b < 0x80 {
+						break
+					}
+				}
+				m.Values = append(m.Values, v)
+			} else if wireType == 2 {
+				var packedLen int
+				for shift := uint(0); ; shift += 7 {
+					if shift >= 64 {
+						return ErrIntOverflowPrivate
+					}
+					if iNdEx >= l {
+						return io.ErrUnexpectedEOF
+					}
+					b := dAtA[iNdEx]
+					iNdEx++
+					packedLen |= int(b&0x7F) << shift
+					if b < 0x80 {
+						break
+					}
+				}
+				if packedLen < 0 {
+					return ErrInvalidLengthPrivate
+				}
+				postIndex := iNdEx + packedLen
+				if postIndex < 0 {
+					return ErrInvalidLengthPrivate
+				}
+				if postIndex > l {
+					return io.ErrUnexpectedEOF
+				}
+				var elementCount int
+				var count int
+				for _, integer := range dAtA[iNdEx:postIndex] {
+					if integer < 128 {
+						count++
+					}
+				}
+				elementCount = count
+				if elementCount != 0 && len(m.Values) == 0 {
+					m.Values = make([]uint64, 0, elementCount)
+				}
+				for iNdEx < postIndex {
+					var v uint64
+					for shift := uint(0); ; shift += 7 {
+						if shift >= 64 {
+							return ErrIntOverflowPrivate
+						}
+						if iNdEx >= l {
+							return io.ErrUnexpectedEOF
+						}
+						b := dAtA[iNdEx]
+						iNdEx++
+						v |= uint64(b&0x7F) << shift
+						if b < 0x80 {
+							break
+						}
+					}
+					m.Values = append(m.Values, v)
+				}
+			} else {
+				return fmt.Errorf("proto: wrong wireType = %d for field Values", wireType)
+			}
+		case 3:
+			if wireType == 0 {
+				var v int64
+				for shift := uint(0); ; shift += 7 {
+					if shift >= 64 {
+						return ErrIntOverflowPrivate
+					}
+					if iNdEx >= l {
+						return io.ErrUnexpectedEOF
+					}
+					b := dAtA[iNdEx]
+					iNdEx++
+					v |= int64(b&0x7F) << shift
+					if b < 0x80 {
+						break
+					}
+				}
+				m.Signed = append(m.Signed, v)
+			} else if wireType == 2 {
+				var packedLen int
+				for shift := uint(0); ; shift += 7 {
+					if shift >= 64 {
+						return ErrIntOverflowPrivate
+					}
+					if iNdEx >= l {
+						return io.ErrUnexpectedEOF
+					}
+					b := dAtA[iNdEx]
+					iNdEx++
+					packedLen |= int(b&0x7F) << shift
+					if b < 0x80 {
+						break
+					}
+				}
+				if packedLen < 0 {
+					return ErrInvalidLengthPrivate
+				}
+				postIndex := iNdEx + packedLen
+				if postIndex < 0 {
+					return ErrInvalidLengthPrivate
+				}
+				if postIndex > l {
+					return io.ErrUnexpectedEOF
+				}
+				var elementCount int
+				var count int
+				for _, integer := range dAtA[iNdEx:postIndex] {
+					if integer < 128 {
+						count++
+					}
+				}
+				elementCount = count
+				if elementCount != 0 && len(m.Signed) == 0 {
+					m.Signed = make([]int64, 0, elementCount)
+				}
+				for iNdEx < postIndex {
+					var v int64
+					for shift := uint(0); ; shift += 7 {
+						if shift >= 64 {
+							return ErrIntOverflowPrivate
+						}
+						if iNdEx >= l {
+							return io.ErrUnexpectedEOF
+						}
+						b := dAtA[iNdEx]
+						iNdEx++
+						v |= int64(b&0x7F) << shift
+						if b < 0x80 {
+							break
+						}
+					}
+					m.Signed = append(m.Signed, v)
+				}
+			} else {
+				return fmt.Errorf("proto: wrong wireType = %d for field Signed", wireType)
+			}
+		default:
+			iNdEx = preIndex
+			skippy, err := skipPrivate(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
+				return ErrInvalidLengthPrivate
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func (m *ShardIngestOperation) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowPrivate
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: ShardIngestOperation: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: ShardIngestOperation: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field OpType", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPrivate
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthPrivate
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthPrivate
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.OpType = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		case 2:
+			if wireType == 0 {
+				var v uint64
+				for shift := uint(0); ; shift += 7 {
+					if shift >= 64 {
+						return ErrIntOverflowPrivate
+					}
+					if iNdEx >= l {
+						return io.ErrUnexpectedEOF
+					}
+					b := dAtA[iNdEx]
+					iNdEx++
+					v |= uint64(b&0x7F) << shift
+					if b < 0x80 {
+						break
+					}
+				}
+				m.ClearRecordIDs = append(m.ClearRecordIDs, v)
+			} else if wireType == 2 {
+				var packedLen int
+				for shift := uint(0); ; shift += 7 {
+					if shift >= 64 {
+						return ErrIntOverflowPrivate
+					}
+					if iNdEx >= l {
+						return io.ErrUnexpectedEOF
+					}
+					b := dAtA[iNdEx]
+					iNdEx++
+					packedLen |= int(b&0x7F) << shift
+					if b < 0x80 {
+						break
+					}
+				}
+				if packedLen < 0 {
+					return ErrInvalidLengthPrivate
+				}
+				postIndex := iNdEx + packedLen
+				if postIndex < 0 {
+					return ErrInvalidLengthPrivate
+				}
+				if postIndex > l {
+					return io.ErrUnexpectedEOF
+				}
+				var elementCount int
+				var count int
+				for _, integer := range dAtA[iNdEx:postIndex] {
+					if integer < 128 {
+						count++
+					}
+				}
+				elementCount = count
+				if elementCount != 0 && len(m.ClearRecordIDs) == 0 {
+					m.ClearRecordIDs = make([]uint64, 0, elementCount)
+				}
+				for iNdEx < postIndex {
+					var v uint64
+					for shift := uint(0); ; shift += 7 {
+						if shift >= 64 {
+							return ErrIntOverflowPrivate
+						}
+						if iNdEx >= l {
+							return io.ErrUnexpectedEOF
+						}
+						b := dAtA[iNdEx]
+						iNdEx++
+						v |= uint64(b&0x7F) << shift
+						if b < 0x80 {
+							break
+						}
+					}
+					m.ClearRecordIDs = append(m.ClearRecordIDs, v)
+				}
+			} else {
+				return fmt.Errorf("proto: wrong wireType = %d for field ClearRecordIDs", wireType)
+			}
+		case 3:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field ClearFields", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPrivate
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthPrivate
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthPrivate
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.ClearFields = append(m.ClearFields, string(dAtA[iNdEx:postIndex]))
+			iNdEx = postIndex
+		case 4:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field FieldOps", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPrivate
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthPrivate
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthPrivate
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			if m.FieldOps == nil {
+				m.FieldOps = make(map[string]*FieldOperation)
+			}
+			var mapkey string
+			var mapvalue *FieldOperation
+			for iNdEx < postIndex {
+				entryPreIndex := iNdEx
+				var wire uint64
+				for shift := uint(0); ; shift += 7 {
+					if shift >= 64 {
+						return ErrIntOverflowPrivate
+					}
+					if iNdEx >= l {
+						return io.ErrUnexpectedEOF
+					}
+					b := dAtA[iNdEx]
+					iNdEx++
+					wire |= uint64(b&0x7F) << shift
+					if b < 0x80 {
+						break
+					}
+				}
+				fieldNum := int32(wire >> 3)
+				if fieldNum == 1 {
+					var stringLenmapkey uint64
+					for shift := uint(0); ; shift += 7 {
+						if shift >= 64 {
+							return ErrIntOverflowPrivate
+						}
+						if iNdEx >= l {
+							return io.ErrUnexpectedEOF
+						}
+						b := dAtA[iNdEx]
+						iNdEx++
+						stringLenmapkey |= uint64(b&0x7F) << shift
+						if b < 0x80 {
+							break
+						}
+					}
+					intStringLenmapkey := int(stringLenmapkey)
+					if intStringLenmapkey < 0 {
+						return ErrInvalidLengthPrivate
+					}
+					postStringIndexmapkey := iNdEx + intStringLenmapkey
+					if postStringIndexmapkey < 0 {
+						return ErrInvalidLengthPrivate
+					}
+					if postStringIndexmapkey > l {
+						return io.ErrUnexpectedEOF
+					}
+					mapkey = string(dAtA[iNdEx:postStringIndexmapkey])
+					iNdEx = postStringIndexmapkey
+				} else if fieldNum == 2 {
+					var mapmsglen int
+					for shift := uint(0); ; shift += 7 {
+						if shift >= 64 {
+							return ErrIntOverflowPrivate
+						}
+						if iNdEx >= l {
+							return io.ErrUnexpectedEOF
+						}
+						b := dAtA[iNdEx]
+						iNdEx++
+						mapmsglen |= int(b&0x7F) << shift
+						if b < 0x80 {
+							break
+						}
+					}
+					if mapmsglen < 0 {
+						return ErrInvalidLengthPrivate
+					}
+					postmsgIndex := iNdEx + mapmsglen
+					if postmsgIndex < 0 {
+						return ErrInvalidLengthPrivate
+					}
+					if postmsgIndex > l {
+						return io.ErrUnexpectedEOF
+					}
+					mapvalue = &FieldOperation{}
+					if err := mapvalue.Unmarshal(dAtA[iNdEx:postmsgIndex]); err != nil {
+						return err
+					}
+					iNdEx = postmsgIndex
+				} else {
+					iNdEx = entryPreIndex
+					skippy, err := skipPrivate(dAtA[iNdEx:])
+					if err != nil {
+						return err
+					}
+					if (skippy < 0) || (iNdEx+skippy) < 0 {
+						return ErrInvalidLengthPrivate
+					}
+					if (iNdEx + skippy) > postIndex {
+						return io.ErrUnexpectedEOF
+					}
+					iNdEx += skippy
+				}
+			}
+			m.FieldOps[mapkey] = mapvalue
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipPrivate(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
+				return ErrInvalidLengthPrivate
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func (m *ShardIngestOperations) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowPrivate
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: ShardIngestOperations: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: ShardIngestOperations: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Ops", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPrivate
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthPrivate
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthPrivate
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Ops = append(m.Ops, &ShardIngestOperation{})
+			if err := m.Ops[len(m.Ops)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipPrivate(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
+				return ErrInvalidLengthPrivate
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func (m *ShardedIngestRequest) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowPrivate
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: ShardedIngestRequest: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: ShardedIngestRequest: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Ops", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPrivate
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthPrivate
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthPrivate
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			if m.Ops == nil {
+				m.Ops = make(map[uint64]*ShardIngestOperations)
+			}
+			var mapkey uint64
+			var mapvalue *ShardIngestOperations
+			for iNdEx < postIndex {
+				entryPreIndex := iNdEx
+				var wire uint64
+				for shift := uint(0); ; shift += 7 {
+					if shift >= 64 {
+						return ErrIntOverflowPrivate
+					}
+					if iNdEx >= l {
+						return io.ErrUnexpectedEOF
+					}
+					b := dAtA[iNdEx]
+					iNdEx++
+					wire |= uint64(b&0x7F) << shift
+					if b < 0x80 {
+						break
+					}
+				}
+				fieldNum := int32(wire >> 3)
+				if fieldNum == 1 {
+					for shift := uint(0); ; shift += 7 {
+						if shift >= 64 {
+							return ErrIntOverflowPrivate
+						}
+						if iNdEx >= l {
+							return io.ErrUnexpectedEOF
+						}
+						b := dAtA[iNdEx]
+						iNdEx++
+						mapkey |= uint64(b&0x7F) << shift
+						if b < 0x80 {
+							break
+						}
+					}
+				} else if fieldNum == 2 {
+					var mapmsglen int
+					for shift := uint(0); ; shift += 7 {
+						if shift >= 64 {
+							return ErrIntOverflowPrivate
+						}
+						if iNdEx >= l {
+							return io.ErrUnexpectedEOF
+						}
+						b := dAtA[iNdEx]
+						iNdEx++
+						mapmsglen |= int(b&0x7F) << shift
+						if b < 0x80 {
+							break
+						}
+					}
+					if mapmsglen < 0 {
+						return ErrInvalidLengthPrivate
+					}
+					postmsgIndex := iNdEx + mapmsglen
+					if postmsgIndex < 0 {
+						return ErrInvalidLengthPrivate
+					}
+					if postmsgIndex > l {
+						return io.ErrUnexpectedEOF
+					}
+					mapvalue = &ShardIngestOperations{}
+					if err := mapvalue.Unmarshal(dAtA[iNdEx:postmsgIndex]); err != nil {
+						return err
+					}
+					iNdEx = postmsgIndex
+				} else {
+					iNdEx = entryPreIndex
+					skippy, err := skipPrivate(dAtA[iNdEx:])
+					if err != nil {
+						return err
+					}
+					if (skippy < 0) || (iNdEx+skippy) < 0 {
+						return ErrInvalidLengthPrivate
+					}
+					if (iNdEx + skippy) > postIndex {
+						return io.ErrUnexpectedEOF
+					}
+					iNdEx += skippy
+				}
+			}
+			m.Ops[mapkey] = mapvalue
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipPrivate(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPrivate
 			}
 			if (iNdEx + skippy) > l {
diff --git a/pb/private.proto b/pb/private.proto
index a5a43af24..9f15939c7 100644
--- a/pb/private.proto
+++ b/pb/private.proto
@@ -234,4 +234,25 @@ message ResizeAbortMessage {
 message ResizeNodeMessage {
 	string NodeID = 1;
 	string Action = 2;
-}
\ No newline at end of file
+}
+
+message FieldOperation {
+    repeated uint64 RecordIDs = 1;
+	repeated uint64 Values = 2;
+	repeated int64 Signed = 3;
+}
+
+message ShardIngestOperation {
+	string OpType = 1;
+	repeated uint64 ClearRecordIDs = 2;
+	repeated string ClearFields = 3;
+	map <string, FieldOperation> FieldOps = 4;
+}
+
+message ShardIngestOperations {
+	repeated ShardIngestOperation Ops = 1;
+}
+
+message ShardedIngestRequest {
+    map <uint64, ShardIngestOperations> Ops = 1;
+}
diff --git a/pb/public.pb.go b/pb/public.pb.go
index 43ebfc641..ca1b84852 100644
--- a/pb/public.pb.go
+++ b/pb/public.pb.go
@@ -5980,10 +5980,7 @@ func (m *Row) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPublic
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPublic
 			}
 			if (iNdEx + skippy) > l {
@@ -6068,10 +6065,7 @@ func (m *RowMatrix) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPublic
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPublic
 			}
 			if (iNdEx + skippy) > l {
@@ -6194,10 +6188,7 @@ func (m *SignedRow) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPublic
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPublic
 			}
 			if (iNdEx + skippy) > l {
@@ -6356,10 +6347,7 @@ func (m *RowIdentifiers) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPublic
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPublic
 			}
 			if (iNdEx + skippy) > l {
@@ -6486,10 +6474,7 @@ func (m *IDList) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPublic
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPublic
 			}
 			if (iNdEx + skippy) > l {
@@ -6593,10 +6578,7 @@ func (m *ExtractedIDColumn) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPublic
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPublic
 			}
 			if (iNdEx + skippy) > l {
@@ -6713,10 +6695,7 @@ func (m *ExtractedIDMatrix) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPublic
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPublic
 			}
 			if (iNdEx + skippy) > l {
@@ -6799,10 +6778,7 @@ func (m *KeyList) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPublic
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPublic
 			}
 			if (iNdEx + skippy) > l {
@@ -7016,10 +6992,7 @@ func (m *ExtractedTableValue) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPublic
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPublic
 			}
 			if (iNdEx + skippy) > l {
@@ -7156,10 +7129,7 @@ func (m *ExtractedTableColumn) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPublic
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPublic
 			}
 			if (iNdEx + skippy) > l {
@@ -7274,10 +7244,7 @@ func (m *ExtractedTableField) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPublic
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPublic
 			}
 			if (iNdEx + skippy) > l {
@@ -7396,10 +7363,7 @@ func (m *ExtractedTable) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPublic
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPublic
 			}
 			if (iNdEx + skippy) > l {
@@ -7520,10 +7484,7 @@ func (m *Pair) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPublic
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPublic
 			}
 			if (iNdEx + skippy) > l {
@@ -7642,10 +7603,7 @@ func (m *PairField) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPublic
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPublic
 			}
 			if (iNdEx + skippy) > l {
@@ -7762,10 +7720,7 @@ func (m *PairsField) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPublic
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPublic
 			}
 			if (iNdEx + skippy) > l {
@@ -7835,10 +7790,7 @@ func (m *Int64) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPublic
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPublic
 			}
 			if (iNdEx + skippy) > l {
@@ -8008,10 +7960,7 @@ func (m *FieldRow) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPublic
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPublic
 			}
 			if (iNdEx + skippy) > l {
@@ -8134,10 +8083,7 @@ func (m *GroupCount) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPublic
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPublic
 			}
 			if (iNdEx + skippy) > l {
@@ -8273,10 +8219,7 @@ func (m *ValCount) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPublic
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPublic
 			}
 			if (iNdEx + skippy) > l {
@@ -8365,10 +8308,7 @@ func (m *Decimal) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPublic
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPublic
 			}
 			if (iNdEx + skippy) > l {
@@ -8620,10 +8560,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPublic
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPublic
 			}
 			if (iNdEx + skippy) > l {
@@ -8740,10 +8677,7 @@ func (m *QueryResponse) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPublic
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPublic
 			}
 			if (iNdEx + skippy) > l {
@@ -9356,10 +9290,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPublic
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPublic
 			}
 			if (iNdEx + skippy) > l {
@@ -9843,10 +9774,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPublic
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPublic
 			}
 			if (iNdEx + skippy) > l {
@@ -10308,10 +10236,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPublic
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPublic
 			}
 			if (iNdEx + skippy) > l {
@@ -10481,10 +10406,7 @@ func (m *AtomicRecord) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPublic
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPublic
 			}
 			if (iNdEx + skippy) > l {
@@ -10567,10 +10489,7 @@ func (m *AtomicImportResponse) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPublic
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPublic
 			}
 			if (iNdEx + skippy) > l {
@@ -10737,10 +10656,7 @@ func (m *TranslateKeysRequest) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPublic
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPublic
 			}
 			if (iNdEx + skippy) > l {
@@ -10867,10 +10783,7 @@ func (m *TranslateKeysResponse) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPublic
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPublic
 			}
 			if (iNdEx + skippy) > l {
@@ -11061,10 +10974,7 @@ func (m *TranslateIDsRequest) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPublic
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPublic
 			}
 			if (iNdEx + skippy) > l {
@@ -11147,10 +11057,7 @@ func (m *TranslateIDsResponse) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPublic
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPublic
 			}
 			if (iNdEx + skippy) > l {
@@ -11267,10 +11174,7 @@ func (m *ImportRoaringRequestView) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPublic
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPublic
 			}
 			if (iNdEx + skippy) > l {
@@ -11484,10 +11388,7 @@ func (m *ImportRoaringRequest) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPublic
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPublic
 			}
 			if (iNdEx + skippy) > l {
@@ -11604,10 +11505,7 @@ func (m *GroupCounts) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthPublic
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthPublic
 			}
 			if (iNdEx + skippy) > l {

From b78ce29a3eeeaa8ee60e61c3768db615be501993 Mon Sep 17 00:00:00 2001
From: Seebs <seebs@molecula.com>
Date: Mon, 27 Sep 2021 11:30:34 -0500
Subject: [PATCH 41/66] unexport ShardedRequest.Merge

This function absolutely shouldn't be used outside of testing, so I've
made the tests using it internal tests and unexported the method.
---
 ingest/op.go      |  2 +-
 ingest/op_test.go | 65 +++++++++++++++++++++++------------------------
 2 files changed, 33 insertions(+), 34 deletions(-)

diff --git a/ingest/op.go b/ingest/op.go
index 754d4e4e2..99d47eeed 100644
--- a/ingest/op.go
+++ b/ingest/op.go
@@ -703,7 +703,7 @@ func (r *Request) ByShard(fields map[string]FieldType) (*ShardedRequest, error)
 
 // merge combines the components of a sharded request back into a single
 // unsharded request, processing shards in numerical order.
-func (s *ShardedRequest) Merge() *Request {
+func (s *ShardedRequest) merge() *Request {
 	req := &Request{}
 	if s == nil || len(s.Ops) == 0 {
 		return req
diff --git a/ingest/op_test.go b/ingest/op_test.go
index db3b5981e..1646e8993 100644
--- a/ingest/op_test.go
+++ b/ingest/op_test.go
@@ -12,30 +12,29 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package ingest_test
+package ingest
 
 import (
 	"math/rand"
 	"testing"
 
-	"github.com/molecula/featurebase/v2/ingest"
 	"github.com/molecula/featurebase/v2/shardwidth"
 )
 
 type opShardingTestCase struct {
 	name   string
-	input  *ingest.Request
-	output *ingest.ShardedRequest
+	input  *Request
+	output *ShardedRequest
 }
 
 var opShardingTestCases = []opShardingTestCase{
 	{
 		name: "sample",
-		input: &ingest.Request{
-			Ops: []*ingest.Operation{
+		input: &Request{
+			Ops: []*Operation{
 				{
-					OpType: ingest.OpSet,
-					FieldOps: map[string]*ingest.FieldOperation{
+					OpType: OpSet,
+					FieldOps: map[string]*FieldOperation{
 						"shard0": {
 							RecordIDs: []uint64{0, 1},
 						},
@@ -54,9 +53,9 @@ var opShardingTestCases = []opShardingTestCase{
 					},
 				},
 				{
-					OpType: ingest.OpRemove,
+					OpType: OpRemove,
 					Seq:    1,
-					FieldOps: map[string]*ingest.FieldOperation{
+					FieldOps: map[string]*FieldOperation{
 						"shard0-2": {
 							RecordIDs: []uint64{1, 2<<shardwidth.Exponent + 1},
 						},
@@ -64,12 +63,12 @@ var opShardingTestCases = []opShardingTestCase{
 				},
 			},
 		},
-		output: &ingest.ShardedRequest{
-			Ops: map[uint64][]*ingest.Operation{
+		output: &ShardedRequest{
+			Ops: map[uint64][]*Operation{
 				0: {
 					{
-						OpType: ingest.OpSet,
-						FieldOps: map[string]*ingest.FieldOperation{
+						OpType: OpSet,
+						FieldOps: map[string]*FieldOperation{
 							"shard0": {
 								RecordIDs: []uint64{0, 1},
 							},
@@ -81,9 +80,9 @@ var opShardingTestCases = []opShardingTestCase{
 						},
 					},
 					{
-						OpType: ingest.OpRemove,
+						OpType: OpRemove,
 						Seq:    1,
-						FieldOps: map[string]*ingest.FieldOperation{
+						FieldOps: map[string]*FieldOperation{
 							"shard0-2": {
 								RecordIDs: []uint64{1},
 							},
@@ -92,8 +91,8 @@ var opShardingTestCases = []opShardingTestCase{
 				},
 				1: {
 					{
-						OpType: ingest.OpSet,
-						FieldOps: map[string]*ingest.FieldOperation{
+						OpType: OpSet,
+						FieldOps: map[string]*FieldOperation{
 							"shard0-1": {
 								RecordIDs: []uint64{
 									1 << shardwidth.Exponent,
@@ -110,9 +109,9 @@ var opShardingTestCases = []opShardingTestCase{
 				},
 				2: {
 					{
-						OpType: ingest.OpRemove,
+						OpType: OpRemove,
 						Seq:    1,
-						FieldOps: map[string]*ingest.FieldOperation{
+						FieldOps: map[string]*FieldOperation{
 							"shard0-2": {
 								RecordIDs: []uint64{2<<shardwidth.Exponent + 1},
 							},
@@ -125,7 +124,7 @@ var opShardingTestCases = []opShardingTestCase{
 }
 
 func TestOpShardingSmall(t *testing.T) {
-	codec, _ := ingest.NewJSONCodec(nil)
+	codec, _ := NewJSONCodec(nil)
 	_ = codec.AddIntField("shard0", nil)
 	_ = codec.AddIntField("shard1", nil)
 	_ = codec.AddIntField("shard0-1", nil)
@@ -140,7 +139,7 @@ func TestOpShardingSmall(t *testing.T) {
 		if err := sharded.Compare(c.output); err != nil {
 			t.Fatalf("%s: shard: %v", c.name, err)
 		}
-		merged := sharded.Merge()
+		merged := sharded.merge()
 		if err := c.input.Compare(merged); err != nil {
 			t.Fatalf("%s: merge: %v", c.name, err)
 		}
@@ -148,7 +147,7 @@ func TestOpShardingSmall(t *testing.T) {
 }
 
 func TestOpShardingLarge(t *testing.T) {
-	codec, err := ingest.NewJSONCodec(nil)
+	codec, err := NewJSONCodec(nil)
 	if err != nil {
 		t.Fatalf("creating codec: %v", err)
 	}
@@ -176,19 +175,19 @@ func TestOpShardingLarge(t *testing.T) {
 		timeStamps[i] = int64(1234567890e9 + (i * 100e9))
 		signedValues[i] = (int64(i) % 16) // no negative values because they won't work with keys
 	}
-	op := &ingest.Operation{
-		OpType:   ingest.OpSet,
-		FieldOps: map[string]*ingest.FieldOperation{},
+	op := &Operation{
+		OpType:   OpSet,
+		FieldOps: map[string]*FieldOperation{},
 	}
-	req := &ingest.Request{
-		Ops: []*ingest.Operation{op},
+	req := &Request{
+		Ops: []*Operation{op},
 	}
-	op.FieldOps["tq"] = &ingest.FieldOperation{
+	op.FieldOps["tq"] = &FieldOperation{
 		RecordIDs: append([]uint64{}, recordIDs...),
 		Values:    append([]uint64{}, values...),
 		Signed:    timeStamps,
 	}
-	op.FieldOps["int"] = &ingest.FieldOperation{
+	op.FieldOps["int"] = &FieldOperation{
 		RecordIDs: recordIDs,
 		Signed:    signedValues,
 	}
@@ -221,7 +220,7 @@ func TestOpShardingLarge(t *testing.T) {
 			}
 		}
 	}
-	op.FieldOps["set"] = &ingest.FieldOperation{
+	op.FieldOps["set"] = &FieldOperation{
 		RecordIDs: recordIDs,
 		Values:    values,
 	}
@@ -230,7 +229,7 @@ func TestOpShardingLarge(t *testing.T) {
 	if err != nil {
 		t.Errorf("sharding: unexpected error %v", err)
 	}
-	merged := sharded.Merge()
+	merged := sharded.merge()
 	if err := req.Compare(merged); err != nil {
 		t.Fatalf("merge comparison: %v", err)
 	}
@@ -241,7 +240,7 @@ func TestFancySharding(t *testing.T) {
 	const recordCount = 5000
 	grr := rand.New(rand.NewSource(0))
 	for i := 0; i < 100; i++ {
-		f := &ingest.FieldOperation{RecordIDs: make([]uint64, recordCount), Values: make([]uint64, recordCount)}
+		f := &FieldOperation{RecordIDs: make([]uint64, recordCount), Values: make([]uint64, recordCount)}
 		shards := make([]int, shardLimit)
 		for j := range f.RecordIDs {
 			v := uint64(grr.Int63n(shardLimit << shardwidth.Exponent))

From 4bab752fa7354cb15c63e4ba7a776ba31e76e54a Mon Sep 17 00:00:00 2001
From: Seebs <seebs@molecula.com>
Date: Mon, 27 Sep 2021 11:31:23 -0500
Subject: [PATCH 42/66] improve comments

---
 api.go | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/api.go b/api.go
index 95e2199bf..1caa34873 100644
--- a/api.go
+++ b/api.go
@@ -1826,11 +1826,18 @@ func (api *API) ImportValueWithTx(ctx context.Context, qcx *Qcx, req *ImportValu
 	return nil
 }
 
-// helper function: do the apply stuff for a known index with known fields
+// ingestNodeOperationsForFields does the actual work of applying operations
+// to a given index with a map of known fields and an already-parsed
+// ShardedRequest. This is used locally on the node that first receives
+// the request, after it does the parsing, and on other nodes because the
+// format they get is already that rather than JSON, so it's the common
+// path *after* key translation and sorting into shards.
 func (api *API) ingestNodeOperationsForFields(ctx context.Context, qcx *Qcx, index *Index, knownFields map[string]*Field, req *ingest.ShardedRequest) error {
 	eg, ctx := errgroup.WithContext(ctx)
 	for shard, ops := range req.Ops {
-		// loop variable shadow capture is the go equivalent of man door hook hand
+		// create new local copies of these values so the goroutine uses these
+		// copies, and doesn't read the actual loop variables, which are being
+		// changed by the loop.
 		shard, ops := shard, ops
 		eg.Go(func() error {
 			return api.applyOperations(ctx, qcx, index, shard, knownFields, ops)

From b41f3554dad170e03780c05c597f410080ca92c1 Mon Sep 17 00:00:00 2001
From: Seebs <seebs@molecula.com>
Date: Mon, 27 Sep 2021 11:34:46 -0500
Subject: [PATCH 43/66] move stableTranslator into test code

It was useful having this in the package to verify code coverage of
the translator, but that having been verified, I'd sort of rather have
it NOT live in the package at all, it's really a testing-only kind
of thing.
---
 ingest/translate.go      | 86 ----------------------------------------
 ingest/translate_test.go | 67 +++++++++++++++++++++++++++++++
 2 files changed, 67 insertions(+), 86 deletions(-)
 delete mode 100644 ingest/translate.go

diff --git a/ingest/translate.go b/ingest/translate.go
deleted file mode 100644
index 42f61e291..000000000
--- a/ingest/translate.go
+++ /dev/null
@@ -1,86 +0,0 @@
-// Copyright 2021 Molecula Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package ingest
-
-import (
-	"fmt"
-)
-
-// stableTranslator implements a key translator that can be reused and
-// will continue to give the same keys for the same values. Possibly
-// surprisingly, it will invent new keys for IDs it is asked about but
-// hasn't seen. This allows us to give a codec which would use keys on
-// translation a request which contains arbitrary numbers, and request
-// text that would parse into that request.
-type stableTranslator struct {
-	in   map[string]uint64
-	out  map[uint64]string
-	next uint64
-}
-
-func (s *stableTranslator) TranslateKeys(keys ...string) (map[string]uint64, error) {
-	ret := make(map[string]uint64, len(keys))
-	for _, key := range keys {
-		if existing, ok := s.in[key]; ok {
-			ret[key] = existing
-			continue
-		}
-		id := s.next
-		// but what if someone already translated that ID, so now it already
-		// exists?
-		if _, ok := s.out[id]; ok {
-			for k := range s.out {
-				if k > id {
-					id = k
-				}
-			}
-			// one larger than the largest we already have. this could
-			// wrap around, in which case, it's your own fault.
-			id++
-		}
-		s.next = id + 1
-		s.in[key] = id
-		s.out[id] = key
-		ret[key] = id
-	}
-	return ret, nil
-}
-
-func (s *stableTranslator) TranslateIDs(ids ...uint64) (map[uint64]string, error) {
-	ret := make(map[uint64]string, len(ids))
-	for _, id := range ids {
-		if existing, ok := s.out[id]; ok {
-			ret[id] = existing
-			continue
-		}
-		key := fmt.Sprintf("k-%d", id)
-		s.in[key] = id
-		s.out[id] = key
-		ret[id] = key
-		if id >= s.next {
-			s.next = id + 1
-		}
-	}
-	return ret, nil
-}
-
-// newStableTranslator produces a translator which can translate forwards
-// and backwards and invent new things if it needs to. Don't use this.
-func newStableTranslator() *stableTranslator {
-	return &stableTranslator{
-		in:  make(map[string]uint64),
-		out: make(map[uint64]string),
-	}
-}
diff --git a/ingest/translate_test.go b/ingest/translate_test.go
index 78deefda7..67daf73c3 100644
--- a/ingest/translate_test.go
+++ b/ingest/translate_test.go
@@ -19,6 +19,73 @@ import (
 	"testing"
 )
 
+// stableTranslator implements a key translator that can be reused and
+// will continue to give the same keys for the same values. Possibly
+// surprisingly, it will invent new keys for IDs it is asked about but
+// hasn't seen. This allows us to give a codec which would use keys on
+// translation a request which contains arbitrary numbers, and request
+// text that would parse into that request.
+type stableTranslator struct {
+	in   map[string]uint64
+	out  map[uint64]string
+	next uint64
+}
+
+func (s *stableTranslator) TranslateKeys(keys ...string) (map[string]uint64, error) {
+	ret := make(map[string]uint64, len(keys))
+	for _, key := range keys {
+		if existing, ok := s.in[key]; ok {
+			ret[key] = existing
+			continue
+		}
+		id := s.next
+		// but what if someone already translated that ID, so now it already
+		// exists?
+		if _, ok := s.out[id]; ok {
+			for k := range s.out {
+				if k > id {
+					id = k
+				}
+			}
+			// one larger than the largest we already have. this could
+			// wrap around, in which case, it's your own fault.
+			id++
+		}
+		s.next = id + 1
+		s.in[key] = id
+		s.out[id] = key
+		ret[key] = id
+	}
+	return ret, nil
+}
+
+func (s *stableTranslator) TranslateIDs(ids ...uint64) (map[uint64]string, error) {
+	ret := make(map[uint64]string, len(ids))
+	for _, id := range ids {
+		if existing, ok := s.out[id]; ok {
+			ret[id] = existing
+			continue
+		}
+		key := fmt.Sprintf("k-%d", id)
+		s.in[key] = id
+		s.out[id] = key
+		ret[id] = key
+		if id >= s.next {
+			s.next = id + 1
+		}
+	}
+	return ret, nil
+}
+
+// newStableTranslator produces a translator which can translate forwards
+// and backwards and invent new things if it needs to. Don't use this.
+func newStableTranslator() *stableTranslator {
+	return &stableTranslator{
+		in:  make(map[string]uint64),
+		out: make(map[uint64]string),
+	}
+}
+
 func TestTranslateReuse(t *testing.T) {
 	// the original stable-translator design had a flaw in that it
 	// assumed that each new ID would always come from a string translation,

From b3f82ac894dfaf535225cc8c2b8afa1b793fb334 Mon Sep 17 00:00:00 2001
From: Seebs <seebs@molecula.com>
Date: Mon, 27 Sep 2021 11:39:19 -0500
Subject: [PATCH 44/66] return early on error instead of writing success status
 also

---
 http/handler.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/http/handler.go b/http/handler.go
index d2eef2e5e..05e97f583 100644
--- a/http/handler.go
+++ b/http/handler.go
@@ -1360,6 +1360,7 @@ func (h *Handler) handlePostIngestData(w http.ResponseWriter, r *http.Request) {
 		err = qcx.Finish()
 		if err != nil {
 			http.Error(w, fmt.Sprintf("ingesting: %v", err), http.StatusInternalServerError)
+			return
 		}
 	} else {
 		qcx.Abort()

From 12882ad14743ff3f377e47ae52d050f45dbaa1fb Mon Sep 17 00:00:00 2001
From: Seebs <seebs@molecula.com>
Date: Mon, 27 Sep 2021 11:54:02 -0500
Subject: [PATCH 45/66] handle replication

I assumed the existing import code handled replicas. It doesn't, actually.
It just assumes they're handled. So, in the new import code, when splitting
things up by-shard, send each shard's data to *every* node that has
that shard, not just the first one.
---
 api.go | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/api.go b/api.go
index 1caa34873..ca3506a0e 100644
--- a/api.go
+++ b/api.go
@@ -1942,16 +1942,25 @@ func (api *API) IngestOperations(ctx context.Context, qcx *Qcx, indexName string
 	if len(snap.Nodes) == 1 {
 		return api.ingestNodeOperationsForFields(ctx, qcx, index, knownFields, sharded)
 	}
-	// split up by fields in some way
+	// Created new ShardedRequest objects for every node, giving each of them
+	// all the shards that apply to them.
 	byNode := make(map[string]*ingest.ShardedRequest)
 	for shard, ops := range sharded.Ops {
 		nodes := snap.ShardNodes(indexName, shard)
-		forThisShard := byNode[nodes[0].ID]
-		if forThisShard == nil {
-			byNode[nodes[0].ID] = &ingest.ShardedRequest{Ops: map[uint64][]*ingest.Operation{shard: ops}}
-			continue
+		for _, node := range nodes {
+			forThisShard := byNode[node.ID]
+			if forThisShard == nil {
+				// Create new ShardedRequest for the target node, with its op map
+				// mapping this shard to the ops for this shard.
+				byNode[node.ID] = &ingest.ShardedRequest{Ops: map[uint64][]*ingest.Operation{shard: ops}}
+				continue
+			}
+			// Add this shard to the existing ShardedRequest's Ops map. Note that
+			// we don't have to worry about overwrites; we can't have seen this
+			// shard before, because we're in a range loop on a map where the shard
+			// is the key.
+			forThisShard.Ops[shard] = ops
 		}
-		forThisShard.Ops[shard] = ops
 	}
 	eg, ctx := errgroup.WithContext(ctx)
 	for _, node := range snap.Nodes {

From 02d3d24bc52ca3e1de591977613a37684c6fd7d4 Mon Sep 17 00:00:00 2001
From: Seebs <seebs@molecula.com>
Date: Mon, 27 Sep 2021 11:58:30 -0500
Subject: [PATCH 46/66] code review cleanup

---
 cluster.go              | 3 +--
 encoding/proto/proto.go | 3 ---
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/cluster.go b/cluster.go
index 29c4c2164..2843b732f 100644
--- a/cluster.go
+++ b/cluster.go
@@ -1567,8 +1567,7 @@ func (c *cluster) translateIndexKeys(ctx context.Context, indexName string, keys
 	return ids, nil
 }
 
-// This implements ingest's key translator interface on a cluster/index
-// pair.
+// This implements ingest's key translator interface on a cluster/index pair.
 type clusterKeyTranslator struct {
 	ctx       context.Context // we're created within a request context and need to pass that to cluster ops
 	c         *cluster
diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go
index 9a8a262a0..c5672e563 100644
--- a/encoding/proto/proto.go
+++ b/encoding/proto/proto.go
@@ -958,9 +958,6 @@ func (s Serializer) encodeShardedIngestRequest(req *ingest.ShardedRequest) *pb.S
 
 func (s Serializer) encodeShardIngestOperations(ops []*ingest.Operation) *pb.ShardIngestOperations {
 	out := &pb.ShardIngestOperations{}
-	if len(ops) == 0 {
-		return out
-	}
 	for _, op := range ops {
 		if op == nil {
 			continue

From 51876b1821d28acf193f47b021c68e9c7678f14e Mon Sep 17 00:00:00 2001
From: Todd Gruben <todd@molecula.com>
Date: Mon, 27 Sep 2021 15:42:07 -0500
Subject: [PATCH 47/66] addsql version to handler

---
 pg/protocol.go | 3 ++-
 pg/query.go    | 1 +
 server.go      | 2 --
 server/pg.go   | 9 +++++++++
 4 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/pg/protocol.go b/pg/protocol.go
index 0fa52a5e9..e58540e1d 100644
--- a/pg/protocol.go
+++ b/pg/protocol.go
@@ -432,7 +432,8 @@ func (p *Portal) Execute() (shouldTerminate bool, queryReady bool, err error) {
 			return
 		}
 		p.Add(rowDescription)
-		dataRow, _ := p.Encoder.TextRow("PostgresSQL 13.0 (molecula)")
+		mesg := fmt.Sprintf("PostgresSQL 13.0 (molecula.%v)", p.server.QueryHandler.Version())
+		dataRow, _ := p.Encoder.TextRow(mesg)
 		p.Add(dataRow)
 	case pgSelect1:
 		rowDescription, e := p.Encoder.EncodeColumn("?column?", int32(23), 4)
diff --git a/pg/query.go b/pg/query.go
index 7aa231670..ebf48e84e 100644
--- a/pg/query.go
+++ b/pg/query.go
@@ -61,6 +61,7 @@ type QueryHandler interface {
 	// HandleQuery executes a query and writes the results back.
 	HandleQuery(context.Context, QueryResultWriter, Query) error
 	HandleSchema(context.Context, *Portal) error
+	Version() string
 }
 
 // queryResultWriter implements QueryResultWrtiter over postgres wire protocol.
diff --git a/server.go b/server.go
index 2e75045d4..93d9947a0 100644
--- a/server.go
+++ b/server.go
@@ -39,7 +39,6 @@ import (
 	"github.com/molecula/featurebase/v2/stats"
 	"github.com/molecula/featurebase/v2/storage"
 	"github.com/molecula/featurebase/v2/topology"
-	"github.com/molecula/featurebase/v2/vprint"
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
 
@@ -1355,7 +1354,6 @@ func (s *Server) PlanSQL(ctx context.Context, q string) (*Stmt, error) {
 	if err != nil {
 		return nil, err
 	}
-	vprint.VV("PLanning SQL: (%v)", q)
 	return NewPlanner(s.executor).PlanStatement(ctx, st)
 }
 
diff --git a/server/pg.go b/server/pg.go
index 8b7d6478f..533ca0b43 100644
--- a/server/pg.go
+++ b/server/pg.go
@@ -596,7 +596,16 @@ func (qdh *QueryDecodeHandler) HandleQuery(ctx context.Context, w pg.QueryResult
 
 	return qdh.Child.HandleQuery(ctx, w, q)
 }
+func (qdh *QueryDecodeHandler) Version() string {
+	return qdh.Child.Version()
+}
 
+func (pqh *PilosaQueryHandler) Version() string {
+	if pqh.sqlVersion > 0 {
+		return "v2"
+	}
+	return "v1"
+}
 func (pqh *PilosaQueryHandler) HandleSchema(ctx context.Context, portal *pg.Portal) error {
 	schema, err := pqh.Api.Schema(context.Background(), false)
 	if err != nil {

From 2b3662508140356950ecf336e34d568415518756 Mon Sep 17 00:00:00 2001
From: Todd Gruben <todd@molecula.com>
Date: Mon, 27 Sep 2021 16:04:40 -0500
Subject: [PATCH 48/66] missed test handler

---
 go.mod               | 2 +-
 pg/pgtest/handler.go | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/go.mod b/go.mod
index e590bdd1b..d64a32730 100644
--- a/go.mod
+++ b/go.mod
@@ -54,7 +54,7 @@ require (
 	golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect
 	golang.org/x/text v0.3.5 // indirect
 	google.golang.org/grpc v1.28.0
-	gopkg.in/yaml.v2 v2.3.0 // indirect
+	gopkg.in/yaml.v2 v2.3.0
 	modernc.org/mathutil v1.0.0
 	modernc.org/strutil v1.0.0
 	sigs.k8s.io/yaml v1.2.0 // indirect
diff --git a/pg/pgtest/handler.go b/pg/pgtest/handler.go
index 02b1361e2..845aedf19 100644
--- a/pg/pgtest/handler.go
+++ b/pg/pgtest/handler.go
@@ -33,6 +33,9 @@ func (h HandlerFunc) HandleQuery(ctx context.Context, w pg.QueryResultWriter, q
 func (h HandlerFunc) HandleSchema(ctx context.Context, portal *pg.Portal) error {
 	return nil
 }
+func (h HandlerFunc) Version() string {
+	return "testv1"
+}
 
 var _ pg.QueryHandler = HandlerFunc(nil)
 

From 79c066a9a01dd6b48b63896ccde5c5e0d9c6ca57 Mon Sep 17 00:00:00 2001
From: Todd Gruben <todd@molecula.com>
Date: Mon, 27 Sep 2021 16:11:27 -0500
Subject: [PATCH 49/66] mod tidy fun

---
 go.mod | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/go.mod b/go.mod
index d64a32730..e590bdd1b 100644
--- a/go.mod
+++ b/go.mod
@@ -54,7 +54,7 @@ require (
 	golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect
 	golang.org/x/text v0.3.5 // indirect
 	google.golang.org/grpc v1.28.0
-	gopkg.in/yaml.v2 v2.3.0
+	gopkg.in/yaml.v2 v2.3.0 // indirect
 	modernc.org/mathutil v1.0.0
 	modernc.org/strutil v1.0.0
 	sigs.k8s.io/yaml v1.2.0 // indirect

From b7126a58598653c48a6aacb1a3890940637e3ee0 Mon Sep 17 00:00:00 2001
From: Ben Johnson <benbjohnson@yahoo.com>
Date: Tue, 28 Sep 2021 13:16:24 -0600
Subject: [PATCH 50/66] Allow StmtRows.Scan() for more types

---
 planner.go | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 66 insertions(+), 2 deletions(-)

diff --git a/planner.go b/planner.go
index 196bfa817..2a418b6c3 100644
--- a/planner.go
+++ b/planner.go
@@ -645,11 +645,12 @@ func (rs *StmtRows) Columns() []*StmtColumn {
 	return rs.node.Columns()
 }
 
-	/*
+/*
 func (rs *StmtRows) Row() int64 {
 	return rs.node.Row()[0].(int64)
 }
-	*/
+*/
+
 func (rs *StmtRows) Next() bool {
 	if rs.err != nil {
 		return false
@@ -696,6 +697,15 @@ func (rs *StmtRows) Scan(dst ...interface{}) error {
 
 		// Copy row value to scan destination.
 		switch v := row[i].(type) {
+		case bool:
+			switch p := dst[i].(type) {
+			case *bool:
+				*p = v
+			case *interface{}:
+				*p = v
+			default:
+				return fmt.Errorf("cannot scan %T value into %T destination at index %d", v, p, i)
+			}
 		case int64:
 			switch p := dst[i].(type) {
 			case *int:
@@ -711,6 +721,48 @@ func (rs *StmtRows) Scan(dst ...interface{}) error {
 			default:
 				return fmt.Errorf("cannot scan %T value into %T destination at index %d", v, p, i)
 			}
+		case uint64:
+			switch p := dst[i].(type) {
+			case *int:
+				*p = int(v)
+			case *int64:
+				*p = int64(v)
+			case *uint:
+				*p = uint(v)
+			case *uint64:
+				*p = uint64(v)
+			case *interface{}:
+				*p = v
+			default:
+				return fmt.Errorf("cannot scan %T value into %T destination at index %d", v, p, i)
+			}
+		case []uint64:
+			switch p := dst[i].(type) {
+			case *[]uint64:
+				*p = []uint64(v)
+			case *interface{}:
+				*p = joinUint64Slice(v)
+			default:
+				return fmt.Errorf("cannot scan %T value into %T destination at index %d", v, p, i)
+			}
+		case string:
+			switch p := dst[i].(type) {
+			case *string:
+				*p = v
+			case *interface{}:
+				*p = v
+			default:
+				return fmt.Errorf("cannot scan %T value into %T destination at index %d", v, p, i)
+			}
+		case []string:
+			switch p := dst[i].(type) {
+			case *[]string:
+				*p = []string(v)
+			case *interface{}:
+				*p = strings.Join(v, ",")
+			default:
+				return fmt.Errorf("cannot scan %T value into %T destination at index %d", v, p, i)
+			}
 		default:
 			return fmt.Errorf("unexpected %T value at index %d", v, i)
 		}
@@ -1113,3 +1165,15 @@ func stringSliceIndex(a []string, v string) int {
 	}
 	return -1
 }
+
+func joinUint64Slice(a []uint64) string {
+	b := []byte("[")
+	for i, v := range a {
+		b = strconv.AppendUint(b, v, 10)
+		if i < len(a)-1 {
+			b = append(b, ',')
+		}
+	}
+	b = append(b, ']')
+	return string(b)
+}

From 3fe28ca7b9b36cd1c55370397979ebe6c049d6a6 Mon Sep 17 00:00:00 2001
From: Souhaila Noor <souhaila.noor@molecula.com>
Date: Tue, 28 Sep 2021 16:34:52 -0500
Subject: [PATCH 51/66] renamed webUI from Pilosa to FeatureBase and updated
 version

---
 Makefile        | 6 +++++-
 http/handler.go | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 4a89aa66c..5f7de2e36 100644
--- a/Makefile
+++ b/Makefile
@@ -3,6 +3,10 @@
 CLONE_URL=github.com/pilosa/pilosa
 MOD_VERSION=v2
 VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
+MAJOR_VERSION=$(shell cut -d '.' -f1 <<<$(VERSION) | cut -d 'v' -f2)
+MAJOR_VERSION_INCREMENT=$(shell echo `expr $(MAJOR_VERSION) + 1`)
+MINOR_VERSION=$(shell cut -d'.' -f2-3 <<<$(VERSION))
+RELEASE_VERSION=$(shell echo v$(MAJOR_VERSION_INCREMENT).$(MINOR_VERSION))
 VARIANT = Molecula
 GO=go
 GOOS=$(shell $(GO) env GOOS)
@@ -13,7 +17,7 @@ BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH)
 BUILD_TIME := $(shell date -u +%FT%T%z)
 SHARD_WIDTH = 20
 COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD)
-LDFLAGS="-X github.com/molecula/featurebase/v2.Version=$(VERSION) -X github.com/molecula/featurebase/v2.BuildTime=$(BUILD_TIME) -X github.com/molecula/featurebase/v2.Variant=$(VARIANT) -X github.com/molecula/featurebase/v2.Commit=$(COMMIT) -X github.com/molecula/featurebase/v2.TrialDeadline=$(TRIAL_DEADLINE)"
+LDFLAGS="-X github.com/molecula/featurebase/v2.Version=$(RELEASE_VERSION) -X github.com/molecula/featurebase/v2.BuildTime=$(BUILD_TIME) -X github.com/molecula/featurebase/v2.Variant=$(VARIANT) -X github.com/molecula/featurebase/v2.Commit=$(COMMIT) -X github.com/molecula/featurebase/v2.TrialDeadline=$(TRIAL_DEADLINE)"
 GO_VERSION=1.16.3
 DOCKER_BUILD= # set to 1 to use `docker-build` instead of `build` when creating a release
 BUILD_TAGS += shardwidth$(SHARD_WIDTH)
diff --git a/http/handler.go b/http/handler.go
index 05e97f583..59a3c3162 100644
--- a/http/handler.go
+++ b/http/handler.go
@@ -527,7 +527,7 @@ func newStatikHandler(h *Handler) statikHandler {
 
 func (s statikHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	if strings.HasPrefix(r.UserAgent(), "curl") {
-		msg := "Welcome. Pilosa v" + s.handler.api.Version() + " is running. Visit https://www.pilosa.com/docs/ for more information."
+		msg := "Welcome. FeatureBase v" + s.handler.api.Version() + " is running. Visit https://docs.molecula.cloud for more information."
 		if s.statikFS != nil {
 			msg += " Try the Web UI by visiting this URL in your browser."
 		}

From 49e8c1c69fbadec52561094f71996ed285768f1f Mon Sep 17 00:00:00 2001
From: Souhaila Noor <souhaila.noor@molecula.com>
Date: Tue, 28 Sep 2021 16:59:54 -0500
Subject: [PATCH 52/66] undid the changes pushed earlier for incrementing the
 release version

---
 Makefile | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/Makefile b/Makefile
index 5f7de2e36..4a89aa66c 100644
--- a/Makefile
+++ b/Makefile
@@ -3,10 +3,6 @@
 CLONE_URL=github.com/pilosa/pilosa
 MOD_VERSION=v2
 VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
-MAJOR_VERSION=$(shell cut -d '.' -f1 <<<$(VERSION) | cut -d 'v' -f2)
-MAJOR_VERSION_INCREMENT=$(shell echo `expr $(MAJOR_VERSION) + 1`)
-MINOR_VERSION=$(shell cut -d'.' -f2-3 <<<$(VERSION))
-RELEASE_VERSION=$(shell echo v$(MAJOR_VERSION_INCREMENT).$(MINOR_VERSION))
 VARIANT = Molecula
 GO=go
 GOOS=$(shell $(GO) env GOOS)
@@ -17,7 +13,7 @@ BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH)
 BUILD_TIME := $(shell date -u +%FT%T%z)
 SHARD_WIDTH = 20
 COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD)
-LDFLAGS="-X github.com/molecula/featurebase/v2.Version=$(RELEASE_VERSION) -X github.com/molecula/featurebase/v2.BuildTime=$(BUILD_TIME) -X github.com/molecula/featurebase/v2.Variant=$(VARIANT) -X github.com/molecula/featurebase/v2.Commit=$(COMMIT) -X github.com/molecula/featurebase/v2.TrialDeadline=$(TRIAL_DEADLINE)"
+LDFLAGS="-X github.com/molecula/featurebase/v2.Version=$(VERSION) -X github.com/molecula/featurebase/v2.BuildTime=$(BUILD_TIME) -X github.com/molecula/featurebase/v2.Variant=$(VARIANT) -X github.com/molecula/featurebase/v2.Commit=$(COMMIT) -X github.com/molecula/featurebase/v2.TrialDeadline=$(TRIAL_DEADLINE)"
 GO_VERSION=1.16.3
 DOCKER_BUILD= # set to 1 to use `docker-build` instead of `build` when creating a release
 BUILD_TAGS += shardwidth$(SHARD_WIDTH)

From 3ef25e4a161579ae3ae1efa1628e2245412480ad Mon Sep 17 00:00:00 2001
From: Seebs <seebs@molecula.com>
Date: Fri, 24 Sep 2021 15:31:33 -0500
Subject: [PATCH 53/66] rework executor's per-shard union to use UnionInPlace

The actual code here is mostly jaffee's, but I've reworked it some.

This doesn't directly seem to be using UnionInPlace, but really it
is.

The actual logic inside (*Row).Union is a mess and probably silly
in a few ways, but hardly matters. The important part is that,
instead of calling it once per child as we get them, we gather
all of them at once and then call it on all of them. That gets
us a call to (*Row).Union that does a very elaborate dance to
compute a call to (*rowSegment).Union on the only segment present
in each of those rows, which then does a simpler thing to
call (*Bitmap).Union() with the first response as a receiver
and the rest as parameters, and THAT then ends up calling either
unionIntoTargetSingle() if there's only one other bitmap,
or using UnionInPlace on a Freeze() of the first bitmap, which
gets us (we hope) the benefits of the fancy UnionInPlace logic.

Every part of this is a reminder that we really need to replace
roaring and also the Row/rowSegment stuff some day.
---
 executor.go | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/executor.go b/executor.go
index a46ca6d2a..ac9f4455e 100644
--- a/executor.go
+++ b/executor.go
@@ -4750,25 +4750,25 @@ func (e *executor) executeIntersectShard(ctx context.Context, qcx *Qcx, index st
 }
 
 // executeUnionShard executes a union() call for a local shard.
-func (e *executor) executeUnionShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (_ *Row, err error) {
+func (e *executor) executeUnionShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (out *Row, err error) {
 	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeUnionShard")
 	defer span.Finish()
 
-	other := NewRow()
+	if len(c.Children) == 0 {
+		return NewRow(), nil
+	}
+	if len(c.Children) == 1 {
+		return e.executeBitmapCallShard(ctx, qcx, index, c.Children[0], shard)
+	}
+	// we have at least two, so...
+	rows := make([]*Row, len(c.Children))
 	for i, input := range c.Children {
-		row, err := e.executeBitmapCallShard(ctx, qcx, index, input, shard)
+		rows[i], err = e.executeBitmapCallShard(ctx, qcx, index, input, shard)
 		if err != nil {
 			return nil, err
 		}
-
-		if i == 0 {
-			other = row
-		} else {
-			other = other.Union(row)
-		}
 	}
-	other.invalidateCount()
-	return other, nil
+	return rows[0].Union(rows[1:]...), nil
 }
 
 // executeXorShard executes a xor() call for a local shard.

From feb8997ca8b638a913a71a4ae4a17350b8545307 Mon Sep 17 00:00:00 2001
From: Samir Patel <48686912+54mir@users.noreply.github.com>
Date: Wed, 29 Sep 2021 15:42:50 -0500
Subject: [PATCH 54/66] Add darwin build to roaring-migrate-tool

---
 cmd/roaring-migrate/ctim_darwin.go | 14 ++++++++++++++
 cmd/roaring-migrate/ctim_linux.go  | 11 +++++++++++
 cmd/roaring-migrate/main.go        |  6 +++---
 3 files changed, 28 insertions(+), 3 deletions(-)
 create mode 100644 cmd/roaring-migrate/ctim_darwin.go
 create mode 100644 cmd/roaring-migrate/ctim_linux.go

diff --git a/cmd/roaring-migrate/ctim_darwin.go b/cmd/roaring-migrate/ctim_darwin.go
new file mode 100644
index 000000000..fe76c74f0
--- /dev/null
+++ b/cmd/roaring-migrate/ctim_darwin.go
@@ -0,0 +1,14 @@
+// +build darwin
+
+package main
+
+import (
+	"syscall"
+	"time"
+)
+
+func CTimeNano(stat *syscall.Stat_t) int64 {
+	ts := stat.Ctimespec
+	time.Unix(int64(ts.Sec), int64(ts.Nsec))
+	return int64(ts.Sec)*1e9 + int64(ts.Nsec)
+}
diff --git a/cmd/roaring-migrate/ctim_linux.go b/cmd/roaring-migrate/ctim_linux.go
new file mode 100644
index 000000000..0ddf61c5d
--- /dev/null
+++ b/cmd/roaring-migrate/ctim_linux.go
@@ -0,0 +1,11 @@
+// +build linux
+
+package main
+
+import (
+	"syscall"
+)
+
+func CTimeNano(stat *syscall.Stat_t) int64 {
+	return stat.Ctim.Nano()
+}
diff --git a/cmd/roaring-migrate/main.go b/cmd/roaring-migrate/main.go
index 60ff5bf7c..8b5b62d6c 100644
--- a/cmd/roaring-migrate/main.go
+++ b/cmd/roaring-migrate/main.go
@@ -24,7 +24,7 @@ import (
 	"strings"
 	"syscall"
 
-	"github.com/molecula/featurebase/v2"
+	pilosa "github.com/molecula/featurebase/v2"
 	"github.com/molecula/featurebase/v2/rbf"
 	"github.com/molecula/featurebase/v2/rbf/cfg"
 	"github.com/molecula/featurebase/v2/roaring"
@@ -148,7 +148,7 @@ func BuildSchema(dataDir string) ([]byte, error) {
 					l = &local{
 						Name:      index,
 						Fields:    make([]*pilosa.FieldInfo, 0),
-						CreatedAt: uint64(stat.Ctim.Nano()),
+						CreatedAt: uint64(CTimeNano(stat)),
 						Options:   *io,
 					}
 					//index options
@@ -158,7 +158,7 @@ func BuildSchema(dataDir string) ([]byte, error) {
 
 				field := t[2]
 				if field != "_exists" {
-					fi, err := pilosa.UnmarshalFieldOptions(field, stat.Ctim.Nano(), content)
+					fi, err := pilosa.UnmarshalFieldOptions(field, CTimeNano(stat), content)
 					if err != nil {
 						return err
 					}

From 5d070b47bb76ca73c2d2c50ae6393d9ea6916b79 Mon Sep 17 00:00:00 2001
From: Samir Patel <48686912+54mir@users.noreply.github.com>
Date: Wed, 29 Sep 2021 15:57:31 -0500
Subject: [PATCH 55/66] clean up

---
 cmd/roaring-migrate/ctim_darwin.go | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/cmd/roaring-migrate/ctim_darwin.go b/cmd/roaring-migrate/ctim_darwin.go
index fe76c74f0..f79dca111 100644
--- a/cmd/roaring-migrate/ctim_darwin.go
+++ b/cmd/roaring-migrate/ctim_darwin.go
@@ -4,11 +4,10 @@ package main
 
 import (
 	"syscall"
-	"time"
 )
 
 func CTimeNano(stat *syscall.Stat_t) int64 {
+	NANOS := int64(1e9) // number of nanosecs in 1 sec
 	ts := stat.Ctimespec
-	time.Unix(int64(ts.Sec), int64(ts.Nsec))
-	return int64(ts.Sec)*1e9 + int64(ts.Nsec)
+	return ts.Sec*NANOS + ts.Nsec
 }

From a845d93a255771ad8d552ec1ce9085ead37b23b2 Mon Sep 17 00:00:00 2001
From: Samir Patel <48686912+54mir@users.noreply.github.com>
Date: Thu, 30 Sep 2021 09:28:33 -0500
Subject: [PATCH 56/66] Add license headers

---
 cmd/roaring-migrate/ctim_darwin.go | 14 ++++++++++++++
 cmd/roaring-migrate/ctim_linux.go  | 14 ++++++++++++++
 2 files changed, 28 insertions(+)

diff --git a/cmd/roaring-migrate/ctim_darwin.go b/cmd/roaring-migrate/ctim_darwin.go
index f79dca111..0b50491ab 100644
--- a/cmd/roaring-migrate/ctim_darwin.go
+++ b/cmd/roaring-migrate/ctim_darwin.go
@@ -1,3 +1,17 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 // +build darwin
 
 package main
diff --git a/cmd/roaring-migrate/ctim_linux.go b/cmd/roaring-migrate/ctim_linux.go
index 0ddf61c5d..b9e5d70e1 100644
--- a/cmd/roaring-migrate/ctim_linux.go
+++ b/cmd/roaring-migrate/ctim_linux.go
@@ -1,3 +1,17 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 // +build linux
 
 package main

From c24a5e77ba36c8129738b85ed91e6cb6ebd6d9a8 Mon Sep 17 00:00:00 2001
From: nagamocha3000 <nagamocha@gmail.com>
Date: Thu, 23 Sep 2021 17:43:16 +0300
Subject: [PATCH 57/66] Test paused node picks up once cluster state is back to
 normal

This adds the following test:
1. cluster comes up (node 1,2,3), status normal
2. Pause node 3
3. Insert keys making sure to filter out the keys that will go to the paused node
4. Wait for status to become degraded
5. Unpause node 3
6. Wait for status to get back to normal
7. Check that keys were replicated to all 3 nodes
---
 Dockerfile-clustertests                       |   4 +
 Makefile                                      |   8 -
 internal/clustertests/cluster_test.go         |   2 +-
 .../docker-compose-index-key-replication.yml  |  73 ----
 internal/clustertests/docker-compose.yml      |   7 +
 .../index_key_replication_test.go             | 375 ----------------
 internal/clustertests/pause_node_test.go      | 412 ++++++++++++++++++
 7 files changed, 424 insertions(+), 457 deletions(-)
 delete mode 100644 internal/clustertests/docker-compose-index-key-replication.yml
 delete mode 100644 internal/clustertests/index_key_replication_test.go
 create mode 100644 internal/clustertests/pause_node_test.go

diff --git a/Dockerfile-clustertests b/Dockerfile-clustertests
index 91fde88fb..8c60f11bf 100644
--- a/Dockerfile-clustertests
+++ b/Dockerfile-clustertests
@@ -14,6 +14,10 @@ RUN cd /go/src/github.com/molecula/featurebase \
 ADD https://github.com/alexei-led/pumba/releases/download/0.6.0/pumba_linux_amd64 /pumba
 RUN chmod +x /pumba
 
+# add docker client to pause/unpause nodes
+RUN apt update
+RUN apt install -y docker.io 
+
 RUN cp /go/bin/featurebase /featurebase
 
 COPY LICENSE /LICENSE
diff --git a/Makefile b/Makefile
index 4a89aa66c..130cae3ce 100644
--- a/Makefile
+++ b/Makefile
@@ -138,14 +138,6 @@ clustertests: vendor
 	docker-compose -f $(DOCKER_COMPOSE) build
 	docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1
 
-
-DOCKER_COMPOSE_INDEX_KEY_REPLICATION=internal/clustertests/docker-compose-index-key-replication.yml
-# Check clustertests target for more info
-clustertests-index-key-replication: vendor
-	docker-compose -f $(DOCKER_COMPOSE_INDEX_KEY_REPLICATION) down
-	docker-compose -f $(DOCKER_COMPOSE_INDEX_KEY_REPLICATION) build
-	docker-compose -f $(DOCKER_COMPOSE_INDEX_KEY_REPLICATION) up --exit-code-from=client1
-
 # Like clustertests, but rebuilds all images.
 clustertests-build: vendor
 	docker-compose -f $(DOCKER_COMPOSE) down -v
diff --git a/internal/clustertests/cluster_test.go b/internal/clustertests/cluster_test.go
index faad15337..3726bcc3b 100644
--- a/internal/clustertests/cluster_test.go
+++ b/internal/clustertests/cluster_test.go
@@ -115,7 +115,7 @@ func waitForStatus(t *testing.T, stator func(context.Context) (string, error), s
 		if err != nil {
 			t.Logf("Status (try %d/%d): %v (retrying in %s)", i, n, err, sleep.String())
 		} else {
-			t.Logf("Status (try %d/%d): %s (retrying in %s)", i, n, s, sleep.String())
+			t.Logf("Status (try %d/%d): curr: %s, expect: %s (retrying in %s)", i, n, s, status, sleep.String())
 		}
 		if s == status {
 			return
diff --git a/internal/clustertests/docker-compose-index-key-replication.yml b/internal/clustertests/docker-compose-index-key-replication.yml
deleted file mode 100644
index 6f84d5947..000000000
--- a/internal/clustertests/docker-compose-index-key-replication.yml
+++ /dev/null
@@ -1,73 +0,0 @@
-version: '2'
-services: 
-  pilosa1:
-    build:
-      context: ../..
-      dockerfile: Dockerfile-clustertests
-    image: ptest
-    ports:
-      - "33455:10101"
-    environment:
-      - PILOSA_NAME=pilosa1
-      - PILOSA_ETCD_LISTEN_CLIENT_ADDRESS=http://0.0.0.0:10201
-      - PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS=http://pilosa1:10201
-      - PILOSA_ETCD_LISTEN_PEER_ADDRESS=http://0.0.0.0:10301
-      - PILOSA_ETCD_ADVERTISE_PEER_ADDRESS=http://pilosa1:10301
-      - PILOSA_ETCD_INITIAL_CLUSTER=pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301,pilosa3=http://pilosa3:10301
-      - PILOSA_CLUSTER_REPLICAS=3
-    networks:
-      - pilosanet
-    command:
-      - "/featurebase server --bind pilosa1:10101"
-  pilosa2:
-    build:
-      context: ../..
-      dockerfile: Dockerfile-clustertests
-    image: ptest
-    ports:
-      - "33456:10101"
-    environment:
-      - PILOSA_NAME=pilosa2
-      - PILOSA_ETCD_LISTEN_CLIENT_ADDRESS=http://0.0.0.0:10201
-      - PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS=http://pilosa2:10201
-      - PILOSA_ETCD_LISTEN_PEER_ADDRESS=http://0.0.0.0:10301
-      - PILOSA_ETCD_ADVERTISE_PEER_ADDRESS=http://pilosa2:10301
-      - PILOSA_ETCD_INITIAL_CLUSTER=pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301,pilosa3=http://pilosa3:10301
-      - PILOSA_CLUSTER_REPLICAS=3
-    networks:
-      - pilosanet
-    command:
-      - "/featurebase server --bind pilosa2:10101"
-  pilosa3:
-    build:
-      context: ../..
-      dockerfile: Dockerfile-clustertests
-    image: ptest
-    ports:
-      - "33457:10101"
-    environment:
-      - PILOSA_NAME=pilosa3
-      - PILOSA_ETCD_LISTEN_CLIENT_ADDRESS=http://0.0.0.0:10201
-      - PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS=http://pilosa3:10201
-      - PILOSA_ETCD_LISTEN_PEER_ADDRESS=http://0.0.0.0:10301
-      - PILOSA_ETCD_ADVERTISE_PEER_ADDRESS=http://pilosa3:10301
-      - PILOSA_ETCD_INITIAL_CLUSTER=pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301,pilosa3=http://pilosa3:10301
-      - PILOSA_CLUSTER_REPLICAS=3
-    networks:
-      - pilosanet
-    command:
-      - "/featurebase server --bind pilosa3:10101"
-  client1:
-    build:
-      context: .
-    environment:
-      - ENABLE_PILOSA_CLUSTER_TESTS_FOR_INDEX_KEY_REPLICATION=1
-      - GO111MODULE=on
-    networks:
-      - pilosanet
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
-    command:
-      - "cd /go/src/github.com/molecula/featurebase/ && go test -mod=vendor -v -run=IndexKey -count=1 github.com/molecula/featurebase/v2/internal/clustertests"
-networks: 
-  pilosanet:
diff --git a/internal/clustertests/docker-compose.yml b/internal/clustertests/docker-compose.yml
index 0c1bf33c1..4192be6f8 100644
--- a/internal/clustertests/docker-compose.yml
+++ b/internal/clustertests/docker-compose.yml
@@ -14,6 +14,7 @@ services:
       - PILOSA_ETCD_LISTEN_PEER_ADDRESS=http://0.0.0.0:10301
       - PILOSA_ETCD_ADVERTISE_PEER_ADDRESS=http://pilosa1:10301
       - PILOSA_ETCD_INITIAL_CLUSTER=pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301,pilosa3=http://pilosa3:10301
+      - PILOSA_CLUSTER_REPLICAS=3
     networks:
       - pilosanet
     command:
@@ -32,6 +33,7 @@ services:
       - PILOSA_ETCD_LISTEN_PEER_ADDRESS=http://0.0.0.0:10301
       - PILOSA_ETCD_ADVERTISE_PEER_ADDRESS=http://pilosa2:10301
       - PILOSA_ETCD_INITIAL_CLUSTER=pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301,pilosa3=http://pilosa3:10301
+      - PILOSA_CLUSTER_REPLICAS=3
     networks:
       - pilosanet
     command:
@@ -50,6 +52,7 @@ services:
       - PILOSA_ETCD_LISTEN_PEER_ADDRESS=http://0.0.0.0:10301
       - PILOSA_ETCD_ADVERTISE_PEER_ADDRESS=http://pilosa3:10301
       - PILOSA_ETCD_INITIAL_CLUSTER=pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301,pilosa3=http://pilosa3:10301
+      - PILOSA_CLUSTER_REPLICAS=3
     networks:
       - pilosanet
     command:
@@ -57,6 +60,10 @@ services:
   client1:
     build:
       context: .
+    depends_on:
+        - "pilosa1"
+        - "pilosa2"
+        - "pilosa3"
     environment:
       - ENABLE_PILOSA_CLUSTER_TESTS=1
       - GO111MODULE=on
diff --git a/internal/clustertests/index_key_replication_test.go b/internal/clustertests/index_key_replication_test.go
deleted file mode 100644
index 54c3dfa63..000000000
--- a/internal/clustertests/index_key_replication_test.go
+++ /dev/null
@@ -1,375 +0,0 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package clustertest
-
-import (
-	"context"
-	"crypto/tls"
-	"fmt"
-	"math/rand"
-	"os"
-	"os/exec"
-	"sync"
-	"testing"
-	"time"
-
-	pilosa "github.com/molecula/featurebase/v2"
-	"github.com/molecula/featurebase/v2/disco"
-	"github.com/molecula/featurebase/v2/http"
-	picli "github.com/molecula/featurebase/v2/http"
-	"github.com/molecula/featurebase/v2/net"
-	"github.com/molecula/featurebase/v2/topology"
-)
-
-// index -> key -> ids from all replicas
-type translationRes map[string]map[string][]uint64
-
-func defaultTranslationResults(indexes []string) translationRes {
-	res := make(translationRes)
-	for _, index := range indexes {
-		res[index] = make(map[string][]uint64)
-	}
-	return res
-}
-
-func verify(allRes translationRes, indexes []string, replicasN, count int) error {
-	received := 0
-	allIDsSame := func(ids []uint64) bool {
-		if len(ids) <= 1 {
-			// trivially true
-			return true
-		}
-		id := ids[0]
-		for _, other := range ids {
-			if id != other {
-				return false
-			}
-		}
-		return true
-	}
-	for _, index := range indexes {
-		res, ok := allRes[index]
-		if !ok {
-			return fmt.Errorf("Expected index '%s' but not present", index)
-		}
-		for key, ids := range res {
-			// first id is after successful translation, rest are from
-			// translate readers
-			replicationCount := len(ids) - 1
-			received += replicationCount
-			if replicationCount != replicasN {
-				return fmt.Errorf("Count for ids for key '%s'(%d), index '%s' not equal than replicasN(%d)", key, replicationCount, index, replicasN)
-			}
-			if !allIDsSame(ids) {
-				return fmt.Errorf("Expected all ids for key '%s', index '%s' to be the same across the cluster %+v", key, index, ids)
-			}
-		}
-	}
-
-	if received != count {
-		return fmt.Errorf("Expected %d count of keys, received %d", count, received)
-	}
-
-	return nil
-}
-func getURIsFromAddresses(addrs []string) ([]*net.URI, error) {
-	uris := make([]*net.URI, 0, len(addrs))
-	for _, addr := range addrs {
-		uri, err := net.NewURIFromAddress(addr)
-		if err != nil {
-			return nil, err
-		}
-		uris = append(uris, uri)
-	}
-	return uris, nil
-}
-
-func getClients(addrs []string) ([]*http.InternalClient, error) {
-	clients := make([]*http.InternalClient, 0, len(addrs))
-	for _, addr := range addrs {
-		c, err := picli.NewInternalClient(addr, picli.GetHTTPClient(nil))
-		if err != nil {
-			return nil, err
-		}
-		clients = append(clients, c)
-	}
-	return clients, nil
-}
-
-func genIndexNames(indexCount int) []string {
-	indexNames := make([]string, 0, indexCount)
-	for i := 1; i <= indexCount; i++ {
-		indexNames = append(indexNames, fmt.Sprintf("idx-%d", i))
-	}
-	return indexNames
-}
-
-func parseDuration(t *testing.T, s string) time.Duration {
-	parsed, err := time.ParseDuration(s)
-	if err != nil {
-		t.Fatal(err)
-	}
-	return parsed
-}
-
-func durationToSeconds(d time.Duration) string {
-	s := int(d.Seconds())
-	return fmt.Sprintf("%ds", s)
-}
-
-func TestIndexKeyReplication(t *testing.T) {
-	if os.Getenv("ENABLE_PILOSA_CLUSTER_TESTS_FOR_INDEX_KEY_REPLICATION") != "1" {
-		t.Skip("pilosa cluster tests for index key replication are not enabled")
-	}
-	// configurations for test
-	replicasN := 3
-	indexCount := 4
-	intervalDurationArg := "100ms"
-	totalInsertionDurationArg := "10s"
-	coolOffDurationArg := "5s"
-	numKeysToInsertPerDuration := 100
-	addresses := []string{"pilosa1:10101", "pilosa2:10101", "pilosa3:10101"}
-
-	intervalDuration := parseDuration(t, intervalDurationArg)
-	totalInsertionDuration := parseDuration(t, totalInsertionDurationArg)
-	coolOffDuration := parseDuration(t, coolOffDurationArg)
-
-	indexes := genIndexNames(indexCount)
-	clients, err := getClients(addresses)
-	cli := clients[0]
-	if err != nil {
-		t.Fatalf("on init clients from addresses: %v, %v", addresses, err)
-	}
-	uris, err := getURIsFromAddresses(addresses)
-	if err != nil {
-		t.Fatalf("on init clients from addresses: %v, %v", addresses, err)
-	}
-	ctx := context.Background()
-
-	// create index keyed
-	for _, index := range indexes {
-		err = cli.EnsureIndex(ctx, index, pilosa.IndexOptions{
-			Keys: true,
-		})
-		if err != nil {
-			t.Fatalf("creating/asserting index: %v", err)
-		}
-	}
-
-	// set up index keys to insert
-	keysInserted := 0
-	allRes := defaultTranslationResults(indexes)
-	{
-		ctxForIndexKeyCreation, cancelFurtherInsertions := context.WithCancel(ctx)
-		var wg sync.WaitGroup
-		var mu = &sync.Mutex{}
-		wg.Add(len(indexes))
-		seed := time.Now().UnixNano()
-		t.Logf("start inserting index keys for %v, seed(%v)", totalInsertionDuration, seed)
-		rng := rand.New(rand.NewSource(seed))
-		for _, index := range indexes {
-			translations := allRes[index]
-			go func(index string, translations map[string][]uint64) {
-				keys := make([]string, numKeysToInsertPerDuration)
-				offset := 1
-				ticker := time.NewTicker(intervalDuration)
-				defer func() {
-					ticker.Stop()
-					wg.Done()
-				}()
-				for {
-					select {
-					case <-ticker.C:
-						for i := 0; i < numKeysToInsertPerDuration; i++ {
-							keys[i] = fmt.Sprintf("key-%d", i+offset)
-						}
-						// pick random node to send key creation to
-						r := rng.Intn(len(addresses))
-						cli, uri := clients[r], uris[r]
-
-						// insert index keys
-						transmap, err := cli.CreateIndexKeysNode(ctxForIndexKeyCreation, uri, index, keys...)
-						if err != nil {
-							if err == context.Canceled {
-								return
-							} else {
-								t.Logf("creating index keys for index(%s) send to node(%s): %v", index, uri.String(), err)
-								continue
-							}
-						}
-						for key, id := range transmap {
-							translations[key] = append(translations[key], id)
-						}
-						mu.Lock()
-						keysInserted += len(transmap)
-						mu.Unlock()
-						offset += numKeysToInsertPerDuration
-					case <-ctxForIndexKeyCreation.Done():
-						return
-					}
-				}
-			}(index, translations)
-		}
-		// inject fault
-		pcmd := exec.Command("/pumba", "netem", "--duration",
-			durationToSeconds(totalInsertionDuration),
-			"loss", "--percent", "50", "--correlation", "60",
-			"clustertests_pilosa3_1")
-		pcmd.Stdout = os.Stdout
-		pcmd.Stderr = os.Stderr
-		t.Logf("sending pumba fault injection cmd: %v", pcmd.String())
-		err = pcmd.Start()
-		if err != nil {
-			t.Fatalf("starting pumba command: %v", err)
-		}
-		err = pcmd.Wait()
-		if err != nil {
-			t.Fatalf("waiting on pumba pause cmd: %v", err)
-		}
-
-		// wait for index keys to be created
-		t.Logf("start wait to complete index key creation")
-		time.Sleep(totalInsertionDuration)
-		cancelFurtherInsertions()
-		wg.Wait()
-		t.Logf("done with inserting index keys. Total keys inserted: %d", keysInserted)
-	}
-
-	t.Logf("start cool off period: %v\n", coolOffDuration)
-	time.Sleep(coolOffDuration)
-	t.Log("done with cool off period, waiting for stability")
-	waitForStatus(t, clients[0].Status, string(disco.ClusterStateNormal), 30, 1*time.Second)
-	t.Log("done with waiting for stability, starting verifying persistence of index keys")
-
-	// get all nodes
-	nodes, err := cli.Nodes(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// prepare translate offset maps
-	nodeMaps := make(map[string]pilosa.TranslateOffsetMap)
-	for _, n := range nodes {
-		nodeMaps[n.ID] = make(pilosa.TranslateOffsetMap)
-	}
-	schema, err := cli.Schema(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	for _, indexInfo := range schema {
-		index := indexInfo.Name
-		isKeyed := indexInfo.Options.Keys
-		if !isKeyed {
-			continue
-		}
-
-		partitionN := topology.DefaultPartitionN
-		for partition := 0; partition < partitionN; partition++ {
-			nodes, err := cli.PartitionNodes(ctx, partition)
-			if err != nil {
-				t.Fatal(err)
-			}
-			for _, n := range nodes {
-				m := nodeMaps[n.ID]
-				m.SetIndexPartitionOffset(index, partition, 0)
-			}
-		}
-	}
-
-	// open translate reader
-	readers := make([]pilosa.TranslateEntryReader, 0, len(nodes))
-	closeAllTranslateReaders := func() []error {
-		var errs []error
-		for _, tr := range readers {
-			err := tr.Close()
-			if err != nil {
-				errs = append(errs, err)
-			}
-		}
-		return errs
-	}
-	var tlsConfig *tls.Config = nil
-	var wg sync.WaitGroup
-	entries := make(chan *pilosa.TranslateEntry)
-	for _, n := range nodes {
-		client := http.GetHTTPClient(tlsConfig)
-		nodeURL := n.URI.String()
-		offsets := nodeMaps[n.ID]
-		openTranslateReader := http.GetOpenTranslateReaderFunc(client)
-		tr, err := openTranslateReader(ctx, nodeURL, offsets)
-		if err != nil {
-			t.Fatal(err)
-		}
-		wg.Add(1)
-		readers = append(readers, tr)
-		go func(node *topology.Node, tr pilosa.TranslateEntryReader) {
-			defer func() {
-				wg.Done()
-			}()
-			for {
-				var entry pilosa.TranslateEntry
-				err := tr.ReadEntry(&entry)
-				if err != nil {
-					// TODO also ignore transient http: read on clsoed response body errors
-					// for now just print error
-					if err != context.Canceled {
-						fmt.Printf("node(%s) On read from translate entry reader: %v", node.URI.String(), err)
-					}
-					return
-				}
-				// ignore field keys
-				if entry.Field != "" {
-					continue
-				}
-				entries <- &entry
-			}
-		}(n, tr)
-	}
-
-	// receive all translation entries made
-	count := replicasN * keysInserted
-	i := 0
-	for {
-		entry := <-entries
-		res, ok := allRes[entry.Index]
-		if !ok {
-			// ignore indexes we did not create for this test
-			continue
-		}
-		ids, ok := res[entry.Key]
-		if !ok {
-			// ignore keys we did not insert for the test
-			continue
-		}
-		res[entry.Key] = append(ids, entry.ID)
-		i++
-		if i == count {
-			break
-		}
-	}
-	errs := closeAllTranslateReaders()
-	wg.Wait()
-	for _, err := range errs {
-		if err != nil {
-			t.Errorf("on close translate readers: %v", err)
-		}
-	}
-	close(entries)
-
-	err = verify(allRes, indexes, replicasN, count)
-	if err != nil {
-		t.Fatal(err)
-	}
-}
diff --git a/internal/clustertests/pause_node_test.go b/internal/clustertests/pause_node_test.go
new file mode 100644
index 000000000..e9c4e822a
--- /dev/null
+++ b/internal/clustertests/pause_node_test.go
@@ -0,0 +1,412 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package clustertest
+
+import (
+	"context"
+	"fmt"
+	"hash/fnv"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"testing"
+	"time"
+
+	pilosa "github.com/molecula/featurebase/v2"
+	boltdb "github.com/molecula/featurebase/v2/boltdb"
+	"github.com/molecula/featurebase/v2/disco"
+	"github.com/molecula/featurebase/v2/http"
+	picli "github.com/molecula/featurebase/v2/http"
+	"github.com/molecula/featurebase/v2/net"
+	"github.com/molecula/featurebase/v2/topology"
+	"github.com/pkg/errors"
+)
+
+func sendCmd(cmd string, args ...string) error {
+	pcmd := exec.Command(cmd, args...)
+	pcmd.Stdout = os.Stdout
+	pcmd.Stderr = os.Stderr
+	err := pcmd.Start()
+	if err != nil {
+		return errors.Wrap(err, "starting cmd")
+	}
+	err = pcmd.Wait()
+	if err != nil {
+		return errors.Wrap(err, "waiting on cmd")
+	}
+	return nil
+}
+
+func unpauseNode(node string) error {
+	unpauseArgs := []string{"container", "unpause", "clustertests_" + node + "_1"}
+	return sendCmd("docker", unpauseArgs...)
+}
+
+func pauseNode(node string) error {
+	pauseArgs := []string{"container", "pause", "clustertests_" + node + "_1"}
+	return sendCmd("docker", pauseArgs...)
+}
+
+type keyInserter struct {
+	client *http.InternalClient
+	uri    *net.URI
+	index  string
+	keys   []string
+}
+
+func (ki keyInserter) insertKeys(ctx context.Context) (map[string]uint64, error) {
+	ts, err := ki.client.CreateIndexKeysNode(ctx, ki.uri, ki.index, ki.keys...)
+	return ts, err
+}
+
+func getAddress(node string) string {
+	return node + ":10101"
+}
+
+func getClients(addrs []string) ([]*http.InternalClient, error) {
+	clients := make([]*http.InternalClient, 0, len(addrs))
+	for _, addr := range addrs {
+		c, err := picli.NewInternalClient(addr, picli.GetHTTPClient(nil))
+		if err != nil {
+			return nil, err
+		}
+		clients = append(clients, c)
+	}
+	return clients, nil
+}
+
+func getURIsFromAddresses(addrs []string) ([]*net.URI, error) {
+	uris := make([]*net.URI, 0, len(addrs))
+	for _, addr := range addrs {
+		uri, err := net.NewURIFromAddress(addr)
+		if err != nil {
+			return nil, err
+		}
+		uris = append(uris, uri)
+	}
+	return uris, nil
+}
+
+func readIndexTranslateData(ctx context.Context, client *picli.InternalClient, dirPath, index string, partition int) error {
+	// read translateStore contents from endpoint
+	r, err := client.IndexTranslateDataReader(ctx, index, partition)
+	if err != nil {
+		return err
+	}
+	buf, err := ioutil.ReadAll(r)
+	if err != nil {
+		return err
+	}
+	r.Close()
+
+	// create file and write contents to it
+	filename := strconv.FormatInt(int64(partition), 10)
+	filePath := filepath.Join(dirPath, filename)
+	file, err := os.Create(filePath)
+	if err != nil {
+		return err
+	}
+	_, err = file.Write(buf)
+	if err != nil {
+		file.Close()
+		return err
+	}
+	err = file.Sync()
+	if err != nil {
+		file.Close()
+		return err
+	}
+	file.Close()
+	return nil
+}
+
+func openTranslateStores(dirPath, index string) (map[int]pilosa.TranslateStore, error) {
+	dirEntries, err := ioutil.ReadDir(dirPath)
+	if err != nil {
+		return nil, err
+	}
+
+	// in case of error, close any translateStore that has been opened
+	rollback := make([]pilosa.TranslateStore, 0, topology.DefaultPartitionN)
+	defer func() {
+		for _, ts := range rollback {
+			_ = ts.Close()
+		}
+	}()
+
+	// filter out non-file entries
+	filePaths := make([]string, 0, len(dirEntries))
+	for _, entry := range dirEntries {
+		if entry.Mode().IsDir() {
+			continue
+		}
+		filePath := filepath.Join(dirPath, entry.Name())
+		filePaths = append(filePaths, filePath)
+	}
+
+	translateStores := make(map[int]pilosa.TranslateStore)
+	for _, filePath := range filePaths {
+		// extract partition number from filename
+		file := filepath.Base(filePath)
+		partition, err := strconv.Atoi(file)
+		if err != nil {
+			return nil, err
+		}
+		// open bolt db
+		ts, err := boltdb.OpenTranslateStore(filePath, index, "", partition, topology.DefaultPartitionN)
+		ts.SetReadOnly(true)
+		if err != nil {
+			return nil, err
+		}
+		translateStores[partition] = ts
+		rollback = append(rollback, ts)
+	}
+	rollback = nil
+
+	return translateStores, nil
+}
+
+var errOpRetriable = errors.New("If operation failed on this error, it can be retried")
+
+func verifyNodeHasGivenKeys(ctx context.Context, node, index, dirPath string, keys []string) error {
+	// get client that's connected to node
+	address := getAddress(node)
+	client, err := picli.NewInternalClient(address, picli.GetHTTPClient(nil))
+	if err != nil {
+		return err
+	}
+
+	// create dir to store boltdbs for this node
+	nodeDirPath := filepath.Join(dirPath, node)
+	err = os.Mkdir(nodeDirPath, 0755)
+	if err != nil {
+		return err
+	}
+
+	// read in all the translate stores for each partition
+	for partition := 0; partition < topology.DefaultPartitionN; partition++ {
+		err := readIndexTranslateData(ctx, client, nodeDirPath, index, partition)
+		if err != nil {
+			return err
+		}
+	}
+
+	// open all the translate stores
+	translateStores, err := openTranslateStores(nodeDirPath, index)
+	if err != nil {
+		return err
+	}
+
+	// close all the translate stores on complete
+	defer func() {
+		for _, ts := range translateStores {
+			ts.Close()
+		}
+	}()
+
+	// merge all translations
+	merged := make(map[string]uint64)
+	for _, ts := range translateStores {
+		entries, err := ts.FindKeys(keys...)
+		if err != nil {
+			return err
+		}
+		for key, id := range entries {
+			merged[key] = id
+		}
+	}
+
+	// check that all expected keys present in node
+	if len(merged) != len(keys) {
+		msg := fmt.Sprintf("entries in node %s: %d. keys inserted: %d",
+			node, len(merged), len(keys))
+		return errors.Wrap(errOpRetriable, msg)
+	}
+	for _, k := range keys {
+		if _, ok := merged[k]; !ok {
+			msg := fmt.Sprintf("Key '%s' not present in node %s", k, node)
+			return errors.Wrap(errOpRetriable, msg)
+		}
+	}
+
+	return nil
+}
+
+func genKeys(count, maxTries int, keyToNode func(string) string, filterOut []string) []string {
+	exclusionSet := make(map[string]struct{})
+	for _, n := range filterOut {
+		exclusionSet[n] = struct{}{}
+	}
+	keys := make([]string, 0, count)
+	i := 0
+	for {
+		if i >= maxTries {
+			break
+		}
+		key := fmt.Sprintf("key-%d", i)
+		i++
+		// get primary node for this key
+		node := keyToNode(key)
+		// check if we should exclude this key
+		if _, ok := exclusionSet[node]; ok {
+			continue
+		}
+		// add key
+		keys = append(keys, key)
+		if len(keys) >= count {
+			return keys
+		}
+	}
+	return keys
+}
+
+func TestPauseReplica(t *testing.T) {
+	if os.Getenv("ENABLE_PILOSA_CLUSTER_TESTS") != "1" {
+		t.Skip("pilosa cluster tests for replication when a replica is paused are not enabled")
+	}
+	// configurations for test
+	nodeNames := []string{"pilosa1", "pilosa2", "pilosa3"}
+	nodeToPause := "pilosa3"
+	addresses := make([]string, len(nodeNames))
+	for i, node := range nodeNames {
+		addresses[i] = getAddress(node)
+	}
+	clients, err := getClients(addresses)
+	if err != nil {
+		t.Fatalf("on init clients from addresses: %v, %v", addresses, err)
+	}
+	cli := clients[0]
+	uris, err := getURIsFromAddresses(addresses)
+	if err != nil {
+		t.Fatalf("on init clients from addresses: %v, %v", addresses, err)
+	}
+	uri := uris[0]
+
+	ctx := context.Background()
+	ctx, cancel := context.WithCancel(ctx)
+
+	t.Log("start Client")
+
+	// first achieve normal cluster status
+	waitForStatus(t, cli.Status, string(disco.ClusterStateNormal), 30, 1*time.Second)
+
+	// create keyed index
+	rng := rand.New(rand.NewSource(time.Now().UnixNano()))
+	index := fmt.Sprintf("keyed-index-%d", rng.Int63())
+	err = cli.EnsureIndex(ctx, index, pilosa.IndexOptions{
+		Keys: true,
+	})
+	if err != nil {
+		t.Fatalf("creating/asserting index: %v", err)
+	}
+
+	// generate mapping from partition to primary node
+	partitionToNode := make([]string, topology.DefaultPartitionN)
+	for partition := 0; partition < topology.DefaultPartitionN; partition++ {
+		nodes, err := cli.PartitionNodes(ctx, partition)
+		if err != nil {
+			t.Fatal(err)
+		}
+		partitionToNode[partition] = nodes[0].URI.Host
+	}
+
+	// generate random keys to insert
+	// no guarantee that all keys expected to be generated don't fall
+	// in nodes to be filtered out
+	keyCount := 100
+	maxKeyGenTries := 1000
+	filterOutKeysFromTheseNodes := []string{nodeToPause}
+	keyToNode := func(key string) string {
+		// get partition for this key
+		h := fnv.New64a()
+		_, _ = h.Write([]byte(index))
+		_, _ = h.Write([]byte(key))
+		partition := int(h.Sum64() % uint64(topology.DefaultPartitionN))
+		// get node for this partition
+		return partitionToNode[partition]
+	}
+	keys := genKeys(keyCount, maxKeyGenTries, keyToNode, filterOutKeysFromTheseNodes)
+
+	// pause node
+	t.Logf("pause %s", nodeToPause)
+	err = pauseNode(nodeToPause)
+	if err != nil {
+		t.Fatalf("error on pause node %s: %v", nodeToPause, err)
+	}
+
+	// insert keys
+	t.Log("start insert")
+	ts, err := keyInserter{
+		client: cli,
+		uri:    uri,
+		index:  index,
+		keys:   keys,
+	}.insertKeys(ctx)
+	if err != nil {
+		t.Fatalf("Error: inserting index keys for index(%s) send to node(%s): %v", index, uri.String(), err)
+	}
+	t.Logf("successfully end insert: %v", len(ts))
+
+	// wait for cluster status to be non-normal
+	waitForStatus(t, cli.Status, string(disco.ClusterStateDegraded), 30, 1*time.Second)
+
+	// wait for cluster status to get back to normal
+	t.Logf("unpause %s", nodeToPause)
+	err = unpauseNode(nodeToPause)
+	if err != nil {
+		t.Fatalf("error on unpause node %s: %v", nodeToPause, err)
+	}
+	waitForStatus(t, cli.Status, string(disco.ClusterStateNormal), 30, 1*time.Second)
+
+	// set up directory to store keys
+	basePath := "."
+	keysDirName := "index_keys"
+	dirPath, err := filepath.Abs(basePath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	dirPath = filepath.Join(dirPath, keysDirName)
+	err = os.Mkdir(dirPath, 0755)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	maxRetries := 10
+	durationInBetweenRetries := 5 * time.Second
+	for _, node := range nodeNames {
+		try := 1
+	retries:
+		for {
+			err = verifyNodeHasGivenKeys(ctx, node, index, dirPath, keys)
+			if err == nil {
+				break retries
+			}
+			if try <= maxRetries && errors.Is(err, errOpRetriable) {
+				try++
+				t.Logf("node %s, retry verify key replication: (%d/%d) after %v\n",
+					node, try, maxRetries, durationInBetweenRetries)
+				time.Sleep(durationInBetweenRetries)
+				continue
+			}
+			t.Fatal(errors.Wrap(err, fmt.Sprintf("try: (%d/%d)", try, maxRetries)))
+		}
+	}
+
+	cancel()
+	t.Log("Done")
+}

From d5b61ee8e854aacffb52e89ef6111f57fe09e9a2 Mon Sep 17 00:00:00 2001
From: Seebs <seebs@molecula.com>
Date: Thu, 30 Sep 2021 12:28:33 -0500
Subject: [PATCH 58/66] reduce etcd fsyncs during testing

We disable fsync more consistently in testing, including using
etcd's already-existing UnsafeNoFsync option to disable fsyncs
in the backing store boltdb used by etcd, to reduce runtime of
our tests on MacOS significantly.

Corresponding to this, we update etcd by one patch to pick
up a locally-invented patch which turns out to be nearly-identical
to the upstream fix for "disabling fsync makes boltdb not
even bother to write some data sometimes", which caused crashes
galore.
---
 etcd/embed.go   |  2 ++
 go.mod          | 55 +++++++++++++++++++++++++++++++++++++++----------
 go.sum          |  2 ++
 idalloc_test.go |  2 +-
 test/cluster.go |  2 +-
 test/disco.go   | 19 ++++++++++-------
 6 files changed, 61 insertions(+), 21 deletions(-)

diff --git a/etcd/embed.go b/etcd/embed.go
index 67711d4c1..7da2c9806 100644
--- a/etcd/embed.go
+++ b/etcd/embed.go
@@ -63,6 +63,7 @@ type Options struct {
 	LClientSocket []*net.TCPListener
 
 	BootstrapTimeout time.Duration
+	UnsafeNoFsync    bool `toml:"no-fsync"`
 }
 
 var (
@@ -228,6 +229,7 @@ func parseOptions(opt Options) *embed.Config {
 	cfg.InitialClusterToken = opt.ClusterName
 	cfg.BootstrapTimeout = opt.BootstrapTimeout
 	cfg.LCUrls = types.MustNewURLs([]string{opt.LClientURL})
+	cfg.UnsafeNoFsync = opt.UnsafeNoFsync
 	if opt.AClientURL != "" {
 		cfg.ACUrls = types.MustNewURLs([]string{opt.AClientURL})
 	} else {
diff --git a/go.mod b/go.mod
index e590bdd1b..23401e34f 100644
--- a/go.mod
+++ b/go.mod
@@ -1,63 +1,96 @@
 module github.com/molecula/featurebase/v2
 
-replace go.etcd.io/etcd => github.com/molecula/etcd v0.0.0-20210621160528-2cd93f1df0e7
+replace go.etcd.io/etcd => github.com/molecula/etcd v0.0.0-20210930172242-ad94b354f72c
 
 require (
 	github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d
 	github.com/DataDog/datadog-go v2.2.0+incompatible
 	github.com/HdrHistogram/hdrhistogram-go v1.1.0 // indirect
+	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d
 	github.com/beevik/ntp v0.3.0
 	github.com/benbjohnson/immutable v0.3.0
+	github.com/beorn7/perks v1.0.0
 	github.com/buger/jsonparser v1.1.1
 	github.com/cespare/xxhash v1.1.0
+	github.com/coreos/go-semver v0.3.0
+	github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e
+	github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f
 	github.com/davecgh/go-spew v1.1.1
-	github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect
-	github.com/dustin/go-humanize v1.0.0 // indirect
+	github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f
+	github.com/dgrijalva/jwt-go v3.2.0+incompatible
+	github.com/dustin/go-humanize v1.0.0
 	github.com/fsnotify/fsnotify v1.4.9 // indirect
 	github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 // indirect
 	github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311
+	github.com/go-ole/go-ole v1.2.4
 	github.com/go-test/deep v1.0.7
 	github.com/gogo/protobuf v1.3.2
+	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
 	github.com/golang/protobuf v1.3.3
+	github.com/google/btree v1.0.0
 	github.com/google/go-cmp v0.5.5
-	github.com/google/uuid v1.1.4 // indirect
+	github.com/google/uuid v1.1.4
 	github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect
 	github.com/gorilla/handlers v1.3.0
 	github.com/gorilla/mux v1.7.0
+	github.com/gorilla/websocket v1.4.2
+	github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4
+	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
+	github.com/grpc-ecosystem/grpc-gateway v1.9.5
 	github.com/improbable-eng/grpc-web v0.13.0
+	github.com/jonboulle/clockwork v0.1.0
+	github.com/json-iterator/go v1.1.7
 	github.com/lib/pq v1.8.0
+	github.com/matttproud/golang_protobuf_extensions v1.0.1
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd
+	github.com/modern-go/reflect2 v1.0.1
 	github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b
 	github.com/opentracing/opentracing-go v1.1.0
 	github.com/pelletier/go-toml v1.4.0
 	github.com/pkg/errors v0.9.1
+	github.com/pmezard/go-difflib v1.0.0
 	github.com/prometheus/client_golang v1.0.0
 	github.com/prometheus/client_model v0.1.0
+	github.com/prometheus/common v0.7.0
+	github.com/prometheus/procfs v0.0.2
 	github.com/prometheus/prom2json v1.3.0
 	github.com/rakyll/statik v0.1.7
-	github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect
-	github.com/rs/cors v1.7.0 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237
+	github.com/rs/cors v1.7.0
 	github.com/satori/go.uuid v1.2.0
 	github.com/shirou/gopsutil/v3 v3.20.11
+	github.com/sirupsen/logrus v1.4.2
+	github.com/soheilhy/cmux v0.1.4
 	github.com/spf13/cobra v1.1.1
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.7.1
 	github.com/stretchr/testify v1.7.0
+	github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5
 	github.com/uber/jaeger-client-go v2.25.0+incompatible
-	github.com/uber/jaeger-lib v2.4.0+incompatible // indirect
+	github.com/uber/jaeger-lib v2.4.0+incompatible
+	github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2
 	github.com/zeebo/blake3 v0.1.1
 	go.etcd.io/bbolt v1.3.5
 	go.etcd.io/etcd v0.0.0-20201125193152-8a03d2e9614b
+	go.uber.org/atomic v1.4.0
+	go.uber.org/multierr v1.1.0
+	go.uber.org/zap v1.10.0
+	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
 	golang.org/x/exp v0.0.0-20201008143054-e3b2a7f2fdc7
 	golang.org/x/mod v0.4.2
-	golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 // indirect
+	golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect
-	golang.org/x/text v0.3.5 // indirect
+	golang.org/x/sys v0.0.0-20210510120138-977fb7262007
+	golang.org/x/text v0.3.5
+	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
+	google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a
 	google.golang.org/grpc v1.28.0
 	gopkg.in/yaml.v2 v2.3.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c
 	modernc.org/mathutil v1.0.0
 	modernc.org/strutil v1.0.0
-	sigs.k8s.io/yaml v1.2.0 // indirect
+	sigs.k8s.io/yaml v1.2.0
 	vitess.io/vitess v3.0.0-rc.3.0.20190602171040-12bfde34629c+incompatible
 )
 
diff --git a/go.sum b/go.sum
index 5f4d1be61..5d6ee4b64 100644
--- a/go.sum
+++ b/go.sum
@@ -232,6 +232,8 @@ github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b h1:cZADDaNYM7xn
 github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b/go.mod h1:uXd1BiH7xLmgkhVmspdJLENv6uGWrTL/MQX2TN7Yz9s=
 github.com/molecula/etcd v0.0.0-20210621160528-2cd93f1df0e7 h1:hufElvtCighE0G2VFJYDGWCY8JlmCWZ1FmXvlf25yUQ=
 github.com/molecula/etcd v0.0.0-20210621160528-2cd93f1df0e7/go.mod h1:1X1h4BZ44WjM0LJof1gKKLap1OA4RsicGCDRtACTkLI=
+github.com/molecula/etcd v0.0.0-20210930172242-ad94b354f72c h1:YnU+8kIrr/7IDGtIYncawklAs54EWihED4DBIm+kjAA=
+github.com/molecula/etcd v0.0.0-20210930172242-ad94b354f72c/go.mod h1:1X1h4BZ44WjM0LJof1gKKLap1OA4RsicGCDRtACTkLI=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223 h1:F9x/1yl3T2AeKLr2AMdilSD8+f9bvMnNN8VS5iDtovc=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
diff --git a/idalloc_test.go b/idalloc_test.go
index dff559146..b28a56604 100644
--- a/idalloc_test.go
+++ b/idalloc_test.go
@@ -40,7 +40,7 @@ func TestIDAlloc(t *testing.T) {
 	}()
 
 	// Open bolt.
-	db, err := bolt.Open(f.Name(), 0666, &bolt.Options{Timeout: 1 * time.Second})
+	db, err := bolt.Open(f.Name(), 0666, &bolt.Options{Timeout: 1 * time.Second, NoSync: true})
 	if err != nil {
 		t.Errorf("opening bolt: %v", err)
 		return
diff --git a/test/cluster.go b/test/cluster.go
index 126245a46..137aa6c83 100644
--- a/test/cluster.go
+++ b/test/cluster.go
@@ -607,7 +607,7 @@ func prependTestServerOpts(opts []server.CommandOption) []server.CommandOption {
 			pilosa.OptServerNodeDownRetries(5, 100*time.Millisecond),
 			pilosa.OptServerStorageConfig(&storage.Config{
 				Backend:      pilosa.CurrentBackendOrDefault(),
-				FsyncEnabled: true,
+				FsyncEnabled: false,
 			}),
 		),
 	}
diff --git a/test/disco.go b/test/disco.go
index 7aebd8f89..d00c06a31 100644
--- a/test/disco.go
+++ b/test/disco.go
@@ -111,6 +111,7 @@ func GetPortsGenConfigs(tb testing.TB, nodes []*Command) error {
 			LPeerSocket:      []*net.TCPListener{peerListener},
 			LClientSocket:    []*net.TCPListener{clientListener},
 			BootstrapTimeout: 50 * time.Millisecond,
+			UnsafeNoFsync:    true,
 		}
 		peerUrls[i] = fmt.Sprintf("%s=%s", name, peerURL)
 	}
@@ -144,14 +145,16 @@ func GenPortsConfig(tb testing.TB, ports []Ports) []*server.Config {
 			BindGRPC:     fmt.Sprintf(":%d", ports[i].Grpc),
 			GRPCListener: ports[i].LsnG,
 			Etcd: etcd.Options{
-				Dir:           discoDir,
-				LClientURL:    lClientURL,
-				AClientURL:    lClientURL,
-				LPeerURL:      lPeerURL,
-				APeerURL:      lPeerURL,
-				HeartbeatTTL:  5,
-				LPeerSocket:   []*net.TCPListener{lsnP},
-				LClientSocket: []*net.TCPListener{lsnC},
+				Dir:              discoDir,
+				LClientURL:       lClientURL,
+				AClientURL:       lClientURL,
+				LPeerURL:         lPeerURL,
+				APeerURL:         lPeerURL,
+				HeartbeatTTL:     5,
+				LPeerSocket:      []*net.TCPListener{lsnP},
+				LClientSocket:    []*net.TCPListener{lsnC},
+				BootstrapTimeout: 50 * time.Millisecond,
+				UnsafeNoFsync:    true,
 			},
 		}
 		cfgs[i].Cluster.Name = clusterName

From 9ac4a5a8f47d382e7676e0821cb0fc6a9e8c3309 Mon Sep 17 00:00:00 2001
From: Seebs <seebs@molecula.com>
Date: Thu, 30 Sep 2021 14:40:11 -0500
Subject: [PATCH 59/66] don't necessarily fsync RBF databases even on close
 when fsync is disabled

In test runs, we open, and close, *huge* numbers of databases. Even
the single fsync on close for these ends up being expensive on some
hosts. *cough* Apple. At least in theory, writes delivered to the
disk are just as written whether or not you've hit fsync, as long
as the machine doesn't power off before getting to them. In the
circumstances where we disable fsync, that's fine.

Since we already have an fsync function for "fsync if it's
not disabled", use that.
---
 rbf/db.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rbf/db.go b/rbf/db.go
index 92efffd72..f1623ded8 100644
--- a/rbf/db.go
+++ b/rbf/db.go
@@ -274,7 +274,7 @@ func (db *DB) Close() (err error) {
 
 	// Close writer handler.
 	if db.file != nil {
-		err = db.file.Sync()
+		err = db.fsync(db.file)
 		if err != nil {
 			return
 		}

From e774acb4a075e3e9cdc104f7517a7114f6ffa5b8 Mon Sep 17 00:00:00 2001
From: Seebs <seebs@molecula.com>
Date: Thu, 30 Sep 2021 14:51:12 -0500
Subject: [PATCH 60/66] disable a few more fsyncs in boltdb

boltdb has a couple of places where it fsyncs even when fsync is
disabled, this turns out to cost an amazing amount of time over
several thousand databases in our test run. In theory, they are
rare circumstances compared to updates; in practice, when you
open 256 partition key translation databases per server opened
and most of them never get written to, not so much.
---
 go.mod | 1 +
 go.sum | 5 +++++
 2 files changed, 6 insertions(+)

diff --git a/go.mod b/go.mod
index 23401e34f..ab18495a3 100644
--- a/go.mod
+++ b/go.mod
@@ -1,6 +1,7 @@
 module github.com/molecula/featurebase/v2
 
 replace go.etcd.io/etcd => github.com/molecula/etcd v0.0.0-20210930172242-ad94b354f72c
+replace go.etcd.io/bbolt => github.com/seebs/bbolt v0.0.0-20210930181431-2ea708af0554
 
 require (
 	github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d
diff --git a/go.sum b/go.sum
index 5d6ee4b64..2d81af164 100644
--- a/go.sum
+++ b/go.sum
@@ -287,6 +287,10 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb
 github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/seebs/bbolt v0.0.0-20210930171653-b02e799f10a9 h1:T3XzfA3QYkfNOLAi7p44L8RdGkLB0wnGjb+Uf8bvA9I=
+github.com/seebs/bbolt v0.0.0-20210930171653-b02e799f10a9/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
+github.com/seebs/bbolt v0.0.0-20210930181431-2ea708af0554 h1:88K0ffxhVphUHxlqW4ewOaXdnJByH4LcCuvYfv0QI/M=
+github.com/seebs/bbolt v0.0.0-20210930181431-2ea708af0554/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
 github.com/shirou/gopsutil/v3 v3.20.11 h1:NeVf1K0cgxsWz+N3671ojRptdgzvp7BXL3KV21R0JnA=
 github.com/shirou/gopsutil/v3 v3.20.11/go.mod h1:igHnfak0qnw1biGeI2qKQvu0ZkwvEkUcCLlYhZzdr/4=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
@@ -446,6 +450,7 @@ golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201014080544-cc95f250f6bc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201024232916-9f70ab9862d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

From 214a1492a8d5ff3cf084273ceadfc5e48bfd5edb Mon Sep 17 00:00:00 2001
From: Seebs <seebs@molecula.com>
Date: Thu, 30 Sep 2021 14:41:48 -0500
Subject: [PATCH 61/66] kill off a ton more fsyncs

Performance of tests on MacOS has been atrocious for a while, and
a lot of that is fsync, so we're trying to make that optional.

To test all of this, I modified RBF to panic if anything tried to
open an RBF database without disabling fsync, and ran the tests that
way, and tracked down the various places this could still happen.

There's a lot of places in our tree where we were creating
test holders which were not getting created with fsync disabled, which
results in a surprisingly large number of points at which we end
up calling fsync in tests, which makes tests much slower than they
need to be. There's also a bunch of places where the flags don't get
propagated correctly; for instance, storage.fsync didn't propagate
to the RBFConfig.

We add an "fsync enabled" flag to OpenTranslateStoreFunc, so we can
tell translation stores that we don't need syncing, so the server's
config can be passed on appropriately.

More of the test code that sets things up is correctly configuring
that flag by default.

We also change the barely-used bolt storage backend to support this as
well.

With this done, the only calls to fsync left in a run of `go test -short`
in the top-level directory are from the zap logger in etcd, and consumed
around 0.03 seconds. The overall impact is that `go test -short`
went from "takes enough more than 10 minutes that i don't know how long
it takes" to about 2.5 minutes.
---
 bolt.go                                  | 10 +++++++--
 bolt_test.go                             |  3 ++-
 boltdb/translate.go                      | 26 +++++++++++++-----------
 boltdb/translate_test.go                 |  2 +-
 cluster_internal_test.go                 |  5 ++++-
 dbshard_internal_test.go                 |  1 +
 executor_internal_test.go                |  2 +-
 executor_test.go                         |  2 +-
 field.go                                 |  2 +-
 field_internal_test.go                   |  2 ++
 field_test.go                            |  6 +++---
 fragment_internal_test.go                |  4 ++--
 holder.go                                |  6 +++---
 holder_internal_test.go                  |  3 +++
 holder_test.go                           | 17 +++++++++++++++-
 idalloc.go                               | 14 +++++++------
 index.go                                 |  2 +-
 index_internal_test.go                   |  2 +-
 index_test.go                            |  2 +-
 internal/clustertests/pause_node_test.go |  2 +-
 server.go                                |  3 +++
 server_internal_test.go                  |  4 +++-
 test/holder.go                           |  5 ++++-
 test/index.go                            |  5 ++++-
 translate.go                             |  4 ++--
 utils_internal_test.go                   |  2 +-
 view_internal_test.go                    |  2 +-
 27 files changed, 92 insertions(+), 46 deletions(-)

diff --git a/bolt.go b/bolt.go
index e679b1400..12435abef 100644
--- a/bolt.go
+++ b/bolt.go
@@ -135,8 +135,12 @@ func (r *boltRegistrar) OpenDBWrapper(path string, doAllocZero bool, cfg *storag
 	if !DirExists(path) {
 		PanicOn(os.MkdirAll(dir, 0755))
 	}
+	fsyncEnabled := true
+	if cfg != nil {
+		fsyncEnabled = cfg.FsyncEnabled
+	}
 
-	db, err := bolt.Open(path, 0666, &bolt.Options{Timeout: 5 * time.Second, InitialMmapSize: TxInitialMmapSize})
+	db, err := bolt.Open(path, 0666, &bolt.Options{Timeout: 5 * time.Second, InitialMmapSize: TxInitialMmapSize, NoSync: !fsyncEnabled})
 	if err != nil {
 		return nil, errors.Wrapf(err, fmt.Sprintf("open bolt path '%v'", path))
 	}
@@ -187,6 +191,7 @@ func (r *boltRegistrar) OpenDBWrapper(path string, doAllocZero bool, cfg *storag
 		openTx:      make(map[*BoltTx]bool),
 
 		DeleteEmptyContainer: true,
+		fsyncEnabled:         cfg.FsyncEnabled,
 	}
 	r.unprotectedRegister(w)
 
@@ -226,7 +231,7 @@ func (w *BoltWrapper) CloseDB() error {
 func (w *BoltWrapper) OpenDB() error {
 	w.muDb.Lock()
 	defer w.muDb.Unlock()
-	db, err := bolt.Open(w.path, 0666, &bolt.Options{Timeout: 5 * time.Second, InitialMmapSize: TxInitialMmapSize})
+	db, err := bolt.Open(w.path, 0666, &bolt.Options{Timeout: 5 * time.Second, InitialMmapSize: TxInitialMmapSize, NoSync: !w.fsyncEnabled})
 	if err != nil {
 		return err
 	}
@@ -315,6 +320,7 @@ type BoltWrapper struct {
 	doAllocZero bool
 
 	DeleteEmptyContainer bool
+	fsyncEnabled         bool // for tracking whether our initial config wanted fsync on
 
 	openTx map[*BoltTx]bool
 }
diff --git a/bolt_test.go b/bolt_test.go
index 5c83ae1af..1dc03b7d0 100644
--- a/bolt_test.go
+++ b/bolt_test.go
@@ -21,6 +21,7 @@ import (
 	"testing"
 
 	"github.com/molecula/featurebase/v2/roaring"
+	"github.com/molecula/featurebase/v2/storage"
 	. "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck
 )
 
@@ -88,7 +89,7 @@ func mustOpenEmptyBoltWrapper(path string) (w *BoltWrapper, cleaner func()) {
 	var err error
 	fn := path
 	PanicOn(os.RemoveAll(fn))
-	ww, err := globalBoltReg.OpenDBWrapper(fn, DetectMemAccessPastTx, nil)
+	ww, err := globalBoltReg.OpenDBWrapper(fn, DetectMemAccessPastTx, &storage.Config{FsyncEnabled: false})
 	PanicOn(err)
 	w = ww.(*BoltWrapper)
 
diff --git a/boltdb/translate.go b/boltdb/translate.go
index 076c77f37..35ef2b08f 100644
--- a/boltdb/translate.go
+++ b/boltdb/translate.go
@@ -55,8 +55,8 @@ const (
 )
 
 // OpenTranslateStore opens and initializes a boltdb translation store.
-func OpenTranslateStore(path, index, field string, partitionID, partitionN int) (pilosa.TranslateStore, error) {
-	s := NewTranslateStore(index, field, partitionID, partitionN)
+func OpenTranslateStore(path, index, field string, partitionID, partitionN int, fsyncEnabled bool) (pilosa.TranslateStore, error) {
+	s := NewTranslateStore(index, field, partitionID, partitionN, fsyncEnabled)
 	s.Path = path
 	if err := s.Open(); err != nil {
 		return nil, err
@@ -88,22 +88,24 @@ type TranslateStore struct {
 	once    sync.Once
 	closing chan struct{}
 
-	readOnly    bool
-	writeNotify chan struct{}
+	readOnly     bool
+	fsyncEnabled bool
+	writeNotify  chan struct{}
 
 	// File path to database file.
 	Path string
 }
 
 // NewTranslateStore returns a new instance of TranslateStore.
-func NewTranslateStore(index, field string, partitionID, partitionN int) *TranslateStore {
+func NewTranslateStore(index, field string, partitionID, partitionN int, fsyncEnabled bool) *TranslateStore {
 	return &TranslateStore{
-		index:       index,
-		field:       field,
-		partitionID: partitionID,
-		partitionN:  partitionN,
-		closing:     make(chan struct{}),
-		writeNotify: make(chan struct{}),
+		index:        index,
+		field:        field,
+		partitionID:  partitionID,
+		partitionN:   partitionN,
+		closing:      make(chan struct{}),
+		writeNotify:  make(chan struct{}),
+		fsyncEnabled: fsyncEnabled,
 	}
 }
 
@@ -120,7 +122,7 @@ func (s *TranslateStore) Open() (err error) {
 
 	if err := os.MkdirAll(filepath.Dir(s.Path), 0777); err != nil {
 		return errors.Wrapf(err, "mkdir %s", filepath.Dir(s.Path))
-	} else if s.db, err = bolt.Open(s.Path, 0666, &bolt.Options{Timeout: 1 * time.Second}); err != nil {
+	} else if s.db, err = bolt.Open(s.Path, 0666, &bolt.Options{Timeout: 1 * time.Second, NoSync: !s.fsyncEnabled}); err != nil {
 		return errors.Wrapf(err, "open file: %s", err)
 	}
 
diff --git a/boltdb/translate_test.go b/boltdb/translate_test.go
index a42d68d50..61f2e61c3 100644
--- a/boltdb/translate_test.go
+++ b/boltdb/translate_test.go
@@ -393,7 +393,7 @@ func MustNewTranslateStore(tb testing.TB) *boltdb.TranslateStore {
 		panic(err)
 	}
 
-	s := boltdb.NewTranslateStore("I", "F", 0, topology.DefaultPartitionN)
+	s := boltdb.NewTranslateStore("I", "F", 0, topology.DefaultPartitionN, false)
 	s.Path = f.Name()
 	return s
 }
diff --git a/cluster_internal_test.go b/cluster_internal_test.go
index 8af84efed..cc55ae9aa 100644
--- a/cluster_internal_test.go
+++ b/cluster_internal_test.go
@@ -155,7 +155,10 @@ func newIndexWithTempPath(tb testing.TB, name string) *Index {
 	if err != nil {
 		panic(err)
 	}
-	h := NewHolder(path, nil)
+	cfg := DefaultHolderConfig()
+	cfg.StorageConfig.FsyncEnabled = false
+	cfg.RBFConfig.FsyncEnabled = false
+	h := NewHolder(path, cfg)
 	PanicOn(h.Open())
 	index, err := h.CreateIndex(name, IndexOptions{})
 	testhook.Cleanup(tb, func() {
diff --git a/dbshard_internal_test.go b/dbshard_internal_test.go
index 81d64436c..9482d1504 100644
--- a/dbshard_internal_test.go
+++ b/dbshard_internal_test.go
@@ -330,6 +330,7 @@ func Test_DBPerShard_GetFieldView2Shards_map_from_RBF(t *testing.T) {
 
 	cfg := mustHolderConfig()
 	cfg.StorageConfig.Backend = "rbf"
+	cfg.StorageConfig.FsyncEnabled = false
 	holder := NewHolder(tmpdir, cfg)
 	defer holder.Close()
 
diff --git a/executor_internal_test.go b/executor_internal_test.go
index 4aa37971c..b752957ef 100644
--- a/executor_internal_test.go
+++ b/executor_internal_test.go
@@ -28,7 +28,7 @@ import (
 
 func TestExecutor_TranslateRowsOnBool(t *testing.T) {
 	path, _ := testhook.TempDirInDir(t, *TempDir, "pilosa-executor-")
-	holder := NewHolder(path, nil)
+	holder := NewHolder(path, mustHolderConfig())
 	defer holder.Close()
 
 	e := &executor{
diff --git a/executor_test.go b/executor_test.go
index 414bf47b5..adcf64dbb 100644
--- a/executor_test.go
+++ b/executor_test.go
@@ -6882,7 +6882,7 @@ func TestMissingKeyRegression(t *testing.T) {
 	c := test.MustRunCluster(t, 1, []server.CommandOption{server.OptCommandServerOptions(
 		pilosa.OptServerStorageConfig(&storage.Config{
 			Backend:      "roaring",
-			FsyncEnabled: true,
+			FsyncEnabled: false,
 		}))})
 	defer c.Close()
 
diff --git a/field.go b/field.go
index a6c4b5e75..4edccfb3f 100644
--- a/field.go
+++ b/field.go
@@ -648,7 +648,7 @@ func (f *Field) writeAvailableShards() {
 func (f *Field) applyTranslateStore() error {
 	// Instantiate & open translation store.
 	var err error
-	f.translateStore, err = f.OpenTranslateStore(f.TranslateStorePath(), f.index, f.name, -1, -1)
+	f.translateStore, err = f.OpenTranslateStore(f.TranslateStorePath(), f.index, f.name, -1, -1, f.holder.cfg.StorageConfig.FsyncEnabled)
 	if err != nil {
 		return errors.Wrap(err, "opening field translate store")
 	}
diff --git a/field_internal_test.go b/field_internal_test.go
index 88ca4b7d7..080836baa 100644
--- a/field_internal_test.go
+++ b/field_internal_test.go
@@ -244,6 +244,8 @@ func NewTestField(t testing.TB, opts FieldOption) *TestField {
 
 	cfg := DefaultHolderConfig()
 	cfg.StorageConfig.Backend = CurrentBackendOrDefault()
+	cfg.StorageConfig.FsyncEnabled = false
+	cfg.RBFConfig.FsyncEnabled = false
 	h := NewHolder(path, cfg)
 	PanicOn(h.Open())
 
diff --git a/field_test.go b/field_test.go
index 38056e8ac..0bc3b60cb 100644
--- a/field_test.go
+++ b/field_test.go
@@ -159,7 +159,7 @@ func TestField_NameRestriction(t *testing.T) {
 	if err != nil {
 		panic(err)
 	}
-	field, err := pilosa.NewField(pilosa.NewHolder(path, nil), path, "i", ".meta", pilosa.OptFieldTypeDefault())
+	field, err := pilosa.NewField(pilosa.NewHolder(path, mustHolderConfig()), path, "i", ".meta", pilosa.OptFieldTypeDefault())
 	if field != nil {
 		t.Fatalf("unexpected field name %s", err)
 	}
@@ -192,13 +192,13 @@ func TestField_NameValidation(t *testing.T) {
 		panic(err)
 	}
 	for _, name := range validFieldNames {
-		_, err := pilosa.NewField(pilosa.NewHolder(path, nil), path, "i", name, pilosa.OptFieldTypeDefault())
+		_, err := pilosa.NewField(pilosa.NewHolder(path, mustHolderConfig()), path, "i", name, pilosa.OptFieldTypeDefault())
 		if err != nil {
 			t.Fatalf("unexpected field name: %s %s", name, err)
 		}
 	}
 	for _, name := range invalidFieldNames {
-		_, err := pilosa.NewField(pilosa.NewHolder(path, nil), path, "i", name, pilosa.OptFieldTypeDefault())
+		_, err := pilosa.NewField(pilosa.NewHolder(path, mustHolderConfig()), path, "i", name, pilosa.OptFieldTypeDefault())
 		if err == nil {
 			t.Fatalf("expected error on field name: %s", name)
 		}
diff --git a/fragment_internal_test.go b/fragment_internal_test.go
index a7915fcc3..9b4ac461a 100644
--- a/fragment_internal_test.go
+++ b/fragment_internal_test.go
@@ -3166,7 +3166,7 @@ func BenchmarkImportIntoLargeFragment(b *testing.B) {
 		origF.Close()
 		fi.Close()
 
-		h := NewHolder(fi.Name(), nil)
+		h := NewHolder(fi.Name(), mustHolderConfig())
 		PanicOn(h.Open())
 		idx, err := h.CreateIndex("i", IndexOptions{})
 		PanicOn(err)
@@ -5146,7 +5146,7 @@ func TestImportClearRestart(t *testing.T) {
 
 				PanicOn(tx2.Commit())
 
-				h3 := NewHolder(filepath.Dir(f2.path()), nil)
+				h3 := NewHolder(filepath.Dir(f2.path()), mustHolderConfig())
 				testhook.Cleanup(t, func() {
 					h3.Close()
 				})
diff --git a/holder.go b/holder.go
index 0daf8416d..dea18e3e7 100644
--- a/holder.go
+++ b/holder.go
@@ -109,7 +109,7 @@ type Holder struct {
 	OpenTransactionStore OpenTransactionStoreFunc
 
 	// Func to open the ID allocator.
-	OpenIDAllocator func(string) (*idAllocator, error)
+	OpenIDAllocator func(string, bool) (*idAllocator, error)
 
 	// transactionManager
 	transactionManager *TransactionManager
@@ -241,7 +241,7 @@ func DefaultHolderConfig() *HolderConfig {
 		OpenTranslateStore:   OpenInMemTranslateStore,
 		OpenTranslateReader:  nil,
 		OpenTransactionStore: OpenInMemTransactionStore,
-		OpenIDAllocator:      func(string) (*idAllocator, error) { return &idAllocator{}, nil },
+		OpenIDAllocator:      func(string, bool) (*idAllocator, error) { return &idAllocator{}, nil },
 		TranslationSyncer:    NopTranslationSyncer,
 		Serializer:           GobSerializer,
 		Schemator:            disco.InMemSchemator,
@@ -623,7 +623,7 @@ func (h *Holder) Open() error {
 	h.transactionManager.Log = h.Logger
 
 	// Open ID allocator.
-	h.ida, err = h.OpenIDAllocator(filepath.Join(h.path, "idalloc.db"))
+	h.ida, err = h.OpenIDAllocator(filepath.Join(h.path, "idalloc.db"), h.cfg.StorageConfig.FsyncEnabled)
 	if err != nil {
 		return errors.Wrap(err, "opening ID allocator")
 	}
diff --git a/holder_internal_test.go b/holder_internal_test.go
index c87b6bf73..bd35f0b03 100644
--- a/holder_internal_test.go
+++ b/holder_internal_test.go
@@ -86,6 +86,7 @@ func makeHolder(tb testing.TB, backend string) (*Holder, string, error) {
 	cfg := mustHolderConfig()
 	if backend != "" {
 		cfg.StorageConfig.Backend = backend
+		cfg.StorageConfig.FsyncEnabled = false
 	}
 	h := NewHolder(path, cfg)
 	return h, path, h.Open()
@@ -265,6 +266,8 @@ func mustHolderConfig() *HolderConfig {
 		_ = MustBackendToTxtype(backend)
 		cfg.StorageConfig.Backend = backend
 	}
+	cfg.StorageConfig.FsyncEnabled = false
+	cfg.RBFConfig.FsyncEnabled = false
 	cfg.Schemator = disco.InMemSchemator
 	cfg.Sharder = disco.InMemSharder
 	return cfg
diff --git a/holder_test.go b/holder_test.go
index 6aeeab79d..4807a997e 100644
--- a/holder_test.go
+++ b/holder_test.go
@@ -25,11 +25,26 @@ import (
 	"time"
 
 	"github.com/molecula/featurebase/v2"
+	"github.com/molecula/featurebase/v2/disco"
 	"github.com/molecula/featurebase/v2/pql"
 	"github.com/molecula/featurebase/v2/test"
 	"github.com/pkg/errors"
 )
 
+// mustHolderConfig provides a default test-friendly holder config.
+func mustHolderConfig() *pilosa.HolderConfig {
+	cfg := pilosa.DefaultHolderConfig()
+	if backend := pilosa.CurrentBackend(); backend != "" {
+		_ = pilosa.MustBackendToTxtype(backend)
+		cfg.StorageConfig.Backend = backend
+	}
+	cfg.StorageConfig.FsyncEnabled = false
+	cfg.RBFConfig.FsyncEnabled = false
+	cfg.Schemator = disco.InMemSchemator
+	cfg.Sharder = disco.InMemSharder
+	return cfg
+}
+
 func TestHolder_Open(t *testing.T) {
 	t.Run("ErrIndexPermission", func(t *testing.T) {
 		if os.Geteuid() == 0 {
@@ -283,7 +298,7 @@ func TestHolder_HasData(t *testing.T) {
 		// Note that we are intentionally not using test.NewHolder,
 		// because we want to create a Holder object with an invalid path,
 		// rather than creating a valid holder with a temporary path.
-		h := pilosa.NewHolder("bad-path", nil)
+		h := pilosa.NewHolder("bad-path", mustHolderConfig())
 
 		if ok, err := h.HasData(); ok || err != nil {
 			t.Fatal("expected HasData to return false, no err, but", ok, err)
diff --git a/idalloc.go b/idalloc.go
index 4c6ea423f..ffa07baf8 100644
--- a/idalloc.go
+++ b/idalloc.go
@@ -53,18 +53,20 @@ func (k IDAllocKey) String() string {
 }
 
 type idAllocator struct {
-	db *bolt.DB
+	db           *bolt.DB
+	fsyncEnabled bool
 }
 
-type OpenIDAllocatorFunc func(path string) (*idAllocator, error) // whyyyyyyyyy
+type OpenIDAllocatorFunc func(path string, enableFsync bool) (*idAllocator, error) // whyyyyyyyyy
 
-func OpenIDAllocator(path string) (*idAllocator, error) {
-	db, err := bolt.Open(path, 0666, &bolt.Options{Timeout: 1 * time.Second})
+func OpenIDAllocator(path string, enableFsync bool) (*idAllocator, error) {
+	db, err := bolt.Open(path, 0666, &bolt.Options{Timeout: 1 * time.Second, NoSync: !enableFsync})
 	if err != nil {
 		return nil, err
 	}
-	return &idAllocator{db}, nil
+	return &idAllocator{db: db, fsyncEnabled: enableFsync}, nil
 }
+
 func (ida *idAllocator) Replace(reader io.Reader) error {
 	newFile := ida.db.Path() + ".bak"
 	liveFile := ida.db.Path()
@@ -92,7 +94,7 @@ func (ida *idAllocator) Replace(reader io.Reader) error {
 	} else {
 		_ = os.Remove(liveFile + ".sav")
 	}
-	db, err := bolt.Open(liveFile, 0666, &bolt.Options{Timeout: 1 * time.Second})
+	db, err := bolt.Open(liveFile, 0666, &bolt.Options{Timeout: 1 * time.Second, NoSync: !ida.fsyncEnabled})
 	ida.db = db
 	return err
 }
diff --git a/index.go b/index.go
index 25b7bd5ee..83a4e8e4a 100644
--- a/index.go
+++ b/index.go
@@ -249,7 +249,7 @@ func (i *Index) open(idx *disco.Index) (err error) {
 			partitionID := partitionID
 
 			g.Go(func() error {
-				store, err := i.OpenTranslateStore(i.TranslateStorePath(partitionID), i.name, "", partitionID, i.holder.partitionN)
+				store, err := i.OpenTranslateStore(i.TranslateStorePath(partitionID), i.name, "", partitionID, i.holder.partitionN, i.holder.cfg.StorageConfig.FsyncEnabled)
 				if err != nil {
 					return errors.Wrapf(err, "opening index translate store: partition=%d", partitionID)
 				}
diff --git a/index_internal_test.go b/index_internal_test.go
index a8cd51706..0bf4f6909 100644
--- a/index_internal_test.go
+++ b/index_internal_test.go
@@ -26,7 +26,7 @@ func mustOpenIndex(tb testing.TB, opt IndexOptions) *Index {
 	if err != nil {
 		panic(err)
 	}
-	h := NewHolder(path, nil)
+	h := NewHolder(path, mustHolderConfig())
 	index, err := h.CreateIndex("i", opt)
 	testhook.Cleanup(tb, func() {
 		h.Close()
diff --git a/index_test.go b/index_test.go
index 149133108..890d481d1 100644
--- a/index_test.go
+++ b/index_test.go
@@ -261,7 +261,7 @@ func TestIndex_InvalidName(t *testing.T) {
 	if err != nil {
 		panic(err)
 	}
-	index, err := pilosa.NewIndex(pilosa.NewHolder(path, nil), path, "ABC")
+	index, err := pilosa.NewIndex(pilosa.NewHolder(path, mustHolderConfig()), path, "ABC")
 	if err == nil {
 		t.Fatalf("should have gotten an error on index name with caps")
 	}
diff --git a/internal/clustertests/pause_node_test.go b/internal/clustertests/pause_node_test.go
index e9c4e822a..2de330a74 100644
--- a/internal/clustertests/pause_node_test.go
+++ b/internal/clustertests/pause_node_test.go
@@ -168,7 +168,7 @@ func openTranslateStores(dirPath, index string) (map[int]pilosa.TranslateStore,
 			return nil, err
 		}
 		// open bolt db
-		ts, err := boltdb.OpenTranslateStore(filePath, index, "", partition, topology.DefaultPartitionN)
+		ts, err := boltdb.OpenTranslateStore(filePath, index, "", partition, topology.DefaultPartitionN, false)
 		ts.SetReadOnly(true)
 		if err != nil {
 			return nil, err
diff --git a/server.go b/server.go
index 93d9947a0..fdca4dc39 100644
--- a/server.go
+++ b/server.go
@@ -339,6 +339,9 @@ func OptServerOpenTranslateReader(fn OpenTranslateReaderFunc) ServerOption {
 func OptServerStorageConfig(cfg *storage.Config) ServerOption {
 	return func(s *Server) error {
 		s.holderConfig.StorageConfig = cfg
+		// For historical reasons, RBF's config can ignore the storage config
+		// in some cases.
+		s.holderConfig.RBFConfig.FsyncEnabled = s.holderConfig.StorageConfig.FsyncEnabled
 		return nil
 	}
 }
diff --git a/server_internal_test.go b/server_internal_test.go
index 7fbddf73e..7a1b7b121 100644
--- a/server_internal_test.go
+++ b/server_internal_test.go
@@ -19,6 +19,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/molecula/featurebase/v2/storage"
 	"github.com/molecula/featurebase/v2/testhook"
 )
 
@@ -45,8 +46,9 @@ func TestMonitorAntiEntropyZero(t *testing.T) {
 	if err != nil {
 		t.Fatalf("getting temp dir: %v", err)
 	}
+	cfg := &storage.Config{FsyncEnabled: false, Backend: storage.DefaultBackend}
 	s, err := NewServer(OptServerDataDir(td),
-		OptServerAntiEntropyInterval(0))
+		OptServerAntiEntropyInterval(0), OptServerStorageConfig(cfg))
 	if err != nil {
 		t.Fatalf("making new server: %v", err)
 	}
diff --git a/test/holder.go b/test/holder.go
index 079495488..6eb63efac 100644
--- a/test/holder.go
+++ b/test/holder.go
@@ -38,7 +38,10 @@ func NewHolder(tb testing.TB) *Holder {
 		panic(err)
 	}
 
-	h := &Holder{Holder: pilosa.NewHolder(path, nil)}
+	cfg := pilosa.DefaultHolderConfig()
+	cfg.StorageConfig.FsyncEnabled = false
+	cfg.RBFConfig.FsyncEnabled = false
+	h := &Holder{Holder: pilosa.NewHolder(path, cfg)}
 
 	return h
 }
diff --git a/test/index.go b/test/index.go
index 608bcae1c..46bc7399b 100644
--- a/test/index.go
+++ b/test/index.go
@@ -33,7 +33,10 @@ func newIndex(tb testing.TB) *Index {
 	if err != nil {
 		panic(err)
 	}
-	h := pilosa.NewHolder(path, pilosa.DefaultHolderConfig())
+	cfg := pilosa.DefaultHolderConfig()
+	cfg.StorageConfig.FsyncEnabled = false
+	cfg.RBFConfig.FsyncEnabled = false
+	h := pilosa.NewHolder(path, cfg)
 	testhook.Cleanup(tb, func() {
 		h.Close()
 	})
diff --git a/translate.go b/translate.go
index f449d75be..256dae341 100644
--- a/translate.go
+++ b/translate.go
@@ -197,7 +197,7 @@ TranslatorSummary{
 }
 
 // OpenTranslateStoreFunc represents a function for instantiating and opening a TranslateStore.
-type OpenTranslateStoreFunc func(path, index, field string, partitionID, partitionN int) (TranslateStore, error)
+type OpenTranslateStoreFunc func(path, index, field string, partitionID, partitionN int, fsyncEnabled bool) (TranslateStore, error)
 
 // GenerateNextPartitionedID returns the next ID within the same partition.
 func GenerateNextPartitionedID(index string, prev uint64, partitionID, partitionN int) uint64 {
@@ -407,7 +407,7 @@ var _ OpenTranslateStoreFunc = OpenInMemTranslateStore
 
 // OpenInMemTranslateStore returns a new instance of InMemTranslateStore.
 // Implements OpenTranslateStoreFunc.
-func OpenInMemTranslateStore(rawurl, index, field string, partitionID, partitionN int) (TranslateStore, error) {
+func OpenInMemTranslateStore(rawurl, index, field string, partitionID, partitionN int, fsyncEnabled bool) (TranslateStore, error) {
 	return NewInMemTranslateStore(index, field, partitionID, partitionN), nil
 }
 
diff --git a/utils_internal_test.go b/utils_internal_test.go
index a3a6af2a7..76c3aa66f 100644
--- a/utils_internal_test.go
+++ b/utils_internal_test.go
@@ -117,7 +117,7 @@ func NewTestClusterWithReplication(tb testing.TB, nNodes, nReplicas, partitionN
 	}
 
 	// holder
-	h := NewHolder(path, nil)
+	h := NewHolder(path, mustHolderConfig())
 
 	// cluster
 	availableShardFileFlushDuration.Set(100 * time.Millisecond)
diff --git a/view_internal_test.go b/view_internal_test.go
index 9258cdd27..e7d2036d8 100644
--- a/view_internal_test.go
+++ b/view_internal_test.go
@@ -35,7 +35,7 @@ func mustOpenView(tb testing.TB, index, field, name string) *view {
 		CacheSize: DefaultCacheSize,
 	}
 
-	h := NewHolder(path, nil)
+	h := NewHolder(path, mustHolderConfig())
 	// h needs an *Index so we can call h.Index() and get Index.Txf, in TestView_DeleteFragment
 
 	cim := &CreateIndexMessage{

From 9db87f78d004b979ca2ca4997598540e14182ecc Mon Sep 17 00:00:00 2001
From: Seebs <seebs@molecula.com>
Date: Thu, 30 Sep 2021 15:44:06 -0500
Subject: [PATCH 62/66] fix go.mod/go.sum

---
 go.mod | 54 +++++++++++-------------------------------------------
 go.sum |  8 --------
 2 files changed, 11 insertions(+), 51 deletions(-)

diff --git a/go.mod b/go.mod
index ab18495a3..ff7c1d715 100644
--- a/go.mod
+++ b/go.mod
@@ -1,97 +1,65 @@
 module github.com/molecula/featurebase/v2
 
 replace go.etcd.io/etcd => github.com/molecula/etcd v0.0.0-20210930172242-ad94b354f72c
+
 replace go.etcd.io/bbolt => github.com/seebs/bbolt v0.0.0-20210930181431-2ea708af0554
 
 require (
 	github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d
 	github.com/DataDog/datadog-go v2.2.0+incompatible
 	github.com/HdrHistogram/hdrhistogram-go v1.1.0 // indirect
-	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d
 	github.com/beevik/ntp v0.3.0
 	github.com/benbjohnson/immutable v0.3.0
-	github.com/beorn7/perks v1.0.0
 	github.com/buger/jsonparser v1.1.1
 	github.com/cespare/xxhash v1.1.0
-	github.com/coreos/go-semver v0.3.0
-	github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e
-	github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f
 	github.com/davecgh/go-spew v1.1.1
-	github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f
-	github.com/dgrijalva/jwt-go v3.2.0+incompatible
-	github.com/dustin/go-humanize v1.0.0
+	github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect
+	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/fsnotify/fsnotify v1.4.9 // indirect
 	github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 // indirect
 	github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311
-	github.com/go-ole/go-ole v1.2.4
 	github.com/go-test/deep v1.0.7
 	github.com/gogo/protobuf v1.3.2
-	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
 	github.com/golang/protobuf v1.3.3
-	github.com/google/btree v1.0.0
 	github.com/google/go-cmp v0.5.5
-	github.com/google/uuid v1.1.4
+	github.com/google/uuid v1.1.4 // indirect
 	github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect
 	github.com/gorilla/handlers v1.3.0
 	github.com/gorilla/mux v1.7.0
-	github.com/gorilla/websocket v1.4.2
-	github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4
-	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
-	github.com/grpc-ecosystem/grpc-gateway v1.9.5
 	github.com/improbable-eng/grpc-web v0.13.0
-	github.com/jonboulle/clockwork v0.1.0
-	github.com/json-iterator/go v1.1.7
 	github.com/lib/pq v1.8.0
-	github.com/matttproud/golang_protobuf_extensions v1.0.1
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd
-	github.com/modern-go/reflect2 v1.0.1
 	github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b
 	github.com/opentracing/opentracing-go v1.1.0
 	github.com/pelletier/go-toml v1.4.0
 	github.com/pkg/errors v0.9.1
-	github.com/pmezard/go-difflib v1.0.0
 	github.com/prometheus/client_golang v1.0.0
 	github.com/prometheus/client_model v0.1.0
-	github.com/prometheus/common v0.7.0
-	github.com/prometheus/procfs v0.0.2
 	github.com/prometheus/prom2json v1.3.0
 	github.com/rakyll/statik v0.1.7
-	github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237
-	github.com/rs/cors v1.7.0
+	github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect
+	github.com/rs/cors v1.7.0 // indirect
 	github.com/satori/go.uuid v1.2.0
 	github.com/shirou/gopsutil/v3 v3.20.11
-	github.com/sirupsen/logrus v1.4.2
-	github.com/soheilhy/cmux v0.1.4
 	github.com/spf13/cobra v1.1.1
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.7.1
 	github.com/stretchr/testify v1.7.0
-	github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5
 	github.com/uber/jaeger-client-go v2.25.0+incompatible
-	github.com/uber/jaeger-lib v2.4.0+incompatible
-	github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2
+	github.com/uber/jaeger-lib v2.4.0+incompatible // indirect
 	github.com/zeebo/blake3 v0.1.1
 	go.etcd.io/bbolt v1.3.5
 	go.etcd.io/etcd v0.0.0-20201125193152-8a03d2e9614b
-	go.uber.org/atomic v1.4.0
-	go.uber.org/multierr v1.1.0
-	go.uber.org/zap v1.10.0
-	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
 	golang.org/x/exp v0.0.0-20201008143054-e3b2a7f2fdc7
 	golang.org/x/mod v0.4.2
-	golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4
+	golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20210510120138-977fb7262007
-	golang.org/x/text v0.3.5
-	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
-	google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a
+	golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect
+	golang.org/x/text v0.3.5 // indirect
 	google.golang.org/grpc v1.28.0
 	gopkg.in/yaml.v2 v2.3.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c
 	modernc.org/mathutil v1.0.0
 	modernc.org/strutil v1.0.0
-	sigs.k8s.io/yaml v1.2.0
+	sigs.k8s.io/yaml v1.2.0 // indirect
 	vitess.io/vitess v3.0.0-rc.3.0.20190602171040-12bfde34629c+incompatible
 )
 
diff --git a/go.sum b/go.sum
index 2d81af164..0959b010c 100644
--- a/go.sum
+++ b/go.sum
@@ -230,8 +230,6 @@ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b h1:cZADDaNYM7xn/nklO3g198JerGQjadFuA0ofxBJgK0Y=
 github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b/go.mod h1:uXd1BiH7xLmgkhVmspdJLENv6uGWrTL/MQX2TN7Yz9s=
-github.com/molecula/etcd v0.0.0-20210621160528-2cd93f1df0e7 h1:hufElvtCighE0G2VFJYDGWCY8JlmCWZ1FmXvlf25yUQ=
-github.com/molecula/etcd v0.0.0-20210621160528-2cd93f1df0e7/go.mod h1:1X1h4BZ44WjM0LJof1gKKLap1OA4RsicGCDRtACTkLI=
 github.com/molecula/etcd v0.0.0-20210930172242-ad94b354f72c h1:YnU+8kIrr/7IDGtIYncawklAs54EWihED4DBIm+kjAA=
 github.com/molecula/etcd v0.0.0-20210930172242-ad94b354f72c/go.mod h1:1X1h4BZ44WjM0LJof1gKKLap1OA4RsicGCDRtACTkLI=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223 h1:F9x/1yl3T2AeKLr2AMdilSD8+f9bvMnNN8VS5iDtovc=
@@ -287,8 +285,6 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb
 github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
-github.com/seebs/bbolt v0.0.0-20210930171653-b02e799f10a9 h1:T3XzfA3QYkfNOLAi7p44L8RdGkLB0wnGjb+Uf8bvA9I=
-github.com/seebs/bbolt v0.0.0-20210930171653-b02e799f10a9/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
 github.com/seebs/bbolt v0.0.0-20210930181431-2ea708af0554 h1:88K0ffxhVphUHxlqW4ewOaXdnJByH4LcCuvYfv0QI/M=
 github.com/seebs/bbolt v0.0.0-20210930181431-2ea708af0554/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
 github.com/shirou/gopsutil/v3 v3.20.11 h1:NeVf1K0cgxsWz+N3671ojRptdgzvp7BXL3KV21R0JnA=
@@ -350,9 +346,6 @@ github.com/zeebo/blake3 v0.1.1 h1:Nbsts7DdKThRHHd+YNlqiGlRqGEF2bE2eXN+xQ1hsEs=
 github.com/zeebo/blake3 v0.1.1/go.mod h1:G9pM4qQwjRzF1/v7+vabMj/c5mWpGZ2Wzo3Eb4z0pb4=
 github.com/zeebo/pcg v1.0.0 h1:dt+dx+HvX8g7Un32rY9XWoYnd0NmKmrIzpHF7qiTDj0=
 github.com/zeebo/pcg v1.0.0/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
-go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
-go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
@@ -449,7 +442,6 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201014080544-cc95f250f6bc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

From 8433f81b6877b4ecd268bce74f69724f102a2cf1 Mon Sep 17 00:00:00 2001
From: Seebs <seebs@molecula.com>
Date: Fri, 1 Oct 2021 11:00:55 -0500
Subject: [PATCH 63/66] don't close storage after failing to open cache

If the inner function that handles the open of storage and cache
fails, we close the fragment. If we closeStorage() before that,
then we can try to close the storage again, which causes a panic
when we try to mark the generation as Done again.

I was going to set f.gen = nil after marking it done, but I'm
not feeling safe about that -- there's too many places where
we check things about f.gen, and it seems unsafe. The generation
code should be removed at some point, because it all exists
as a workaround for not having any way to detect when reads are
"done", because we didn't want to do something huge and intrusive,
like adding the Tx system and requiring transactions to get
closed.
---
 fragment.go | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/fragment.go b/fragment.go
index d545a1726..daab8bb34 100644
--- a/fragment.go
+++ b/fragment.go
@@ -286,10 +286,6 @@ func (f *fragment) Open() error {
 		// Fill cache with rows persisted to disk.
 		f.holder.Logger.Debugf("open cache for index/field/view/fragment: %s/%s/%s/%d", f.index(), f.field(), f.view(), f.shard)
 		if err := f.openCache(); err != nil {
-			e2 := f.closeStorage()
-			if e2 != nil {
-				return errors.Wrapf(err, "closing storage: %v, after opening cache", e2)
-			}
 			return errors.Wrap(err, "opening cache")
 		}
 

From 98e7ade5915eeb73f2b901d583f789a4bb1d4adf Mon Sep 17 00:00:00 2001
From: Ben Johnson <benbjohnson@yahoo.com>
Date: Thu, 7 Oct 2021 14:20:07 -0600
Subject: [PATCH 64/66] CORE-809: Aggregate COUNT() with INNER JOIN

---
 planner.go      | 263 +++++++++++++++++++++++++++++++++++++++---------
 planner_test.go |  83 +++++++++++++++
 sql2/ast.go     | 130 ++++++++++++++++++++++++
 3 files changed, 426 insertions(+), 50 deletions(-)

diff --git a/planner.go b/planner.go
index 2a418b6c3..7942b32b4 100644
--- a/planner.go
+++ b/planner.go
@@ -62,6 +62,11 @@ func (p *Planner) planSelectStatement(ctx context.Context, stmt *sql2.SelectStat
 }
 
 func (p *Planner) planAggregateSelectStatement(ctx context.Context, stmt *sql2.SelectStatement) (_ StmtNode, err error) {
+	// Handle specific case of a two-table INNER JOIN with a COUNT().
+	if _, ok := stmt.Source.(*sql2.JoinClause); ok {
+		return p.planAggregateCountJoin(ctx, stmt)
+	}
+
 	indexName, err := statementTableName(stmt)
 	if err != nil {
 		return nil, err
@@ -116,7 +121,13 @@ func (p *Planner) planAggregateSelectStatement(ctx context.Context, stmt *sql2.S
 	switch callName {
 	case "COUNT":
 		if len(groupByCols) == 0 {
-			return NewCountNode(p.executor, indexName, columns[0], cond), nil
+			if cond == nil {
+				cond = &pql.Call{Name: "All"}
+			}
+			return NewCountNode(p.executor, indexName, columns[0], &pql.Call{
+				Name:     "Count",
+				Children: []*pql.Call{cond},
+			}), nil
 		}
 
 		var aggregate *pql.Call
@@ -163,6 +174,156 @@ func (p *Planner) planAggregateSelectStatement(ctx context.Context, stmt *sql2.S
 	// TODO: Support HAVING
 }
 
+func (p *Planner) planAggregateCountJoin(ctx context.Context, stmt *sql2.SelectStatement) (_ StmtNode, err error) {
+	// Ensure we have an INNER JOIN.
+	join := stmt.Source.(*sql2.JoinClause) // caller checked
+	if !join.Operator.Inner.IsValid() {
+		return nil, fmt.Errorf("only inner joins are currently supported")
+	}
+
+	// Determine the two tables we are joining.
+	tbl0, ok := join.X.(*sql2.QualifiedTableName)
+	if !ok {
+		return nil, fmt.Errorf("left side of join must be a table")
+	}
+	tbl1, ok := join.Y.(*sql2.QualifiedTableName)
+	if !ok {
+		return nil, fmt.Errorf("left side of join must be a table")
+	}
+
+	// Ensure INNER JOIN has an "ON" constraint.
+	if join.Constraint == nil {
+		return nil, fmt.Errorf("joins must have an ON constraint")
+	}
+	cons, ok := join.Constraint.(*sql2.OnConstraint)
+	if !ok {
+		return nil, fmt.Errorf("joins only support an ON constraint")
+	}
+
+	// Determine the joined columns.
+	cx, ok := cons.X.(*sql2.BinaryExpr)
+	if !ok {
+		return nil, fmt.Errorf("join must use a binary expression")
+	} else if cx.Op != sql2.EQ {
+		return nil, fmt.Errorf("join must use an equality expression")
+	}
+
+	// Extract join columns & validate that they reference known tables and join on "_id".
+	x, ok := cx.X.(*sql2.QualifiedRef)
+	if !ok {
+		return nil, fmt.Errorf("left-hand side of join expression must be a table-qualified column")
+	} else if x.Table.Name != tbl0.TableName() && x.Table.Name != tbl1.TableName() {
+		return nil, fmt.Errorf("no such table: %q", x.Table.Name)
+	}
+
+	y, ok := cx.Y.(*sql2.QualifiedRef)
+	if !ok {
+		return nil, fmt.Errorf("right-hand side of join expression must be a table-qualified column")
+	} else if y.Table.Name != tbl0.TableName() && y.Table.Name != tbl1.TableName() {
+		return nil, fmt.Errorf("no such table: %q", y.Table.Name)
+	}
+
+	if x.Column.Name != "_id" && y.Column.Name != "_id" {
+		return nil, fmt.Errorf("must join table on _id column")
+	} else if x.Column.Name == "_id" && y.Column.Name == "_id" {
+		return nil, fmt.Errorf("cannot join _id field of two tables")
+	}
+
+	// Move ID column to LHS.
+	if x.Column.Name != "_id" {
+		x, y = y, x
+	}
+
+	// Move parent table to LHS.
+	if x.Table.Name != tbl0.TableName() {
+		tbl0, tbl1 = tbl1, tbl0
+	}
+
+	// Ensure column expression is a single COUNT.
+	if len(stmt.Columns) != 1 {
+		return nil, fmt.Errorf("only COUNT() is supported on joined tables")
+	}
+	expr, ok := stmt.Columns[0].Expr.(*sql2.Call)
+	if !ok || strings.ToUpper(expr.Name.Name) != "COUNT" {
+		return nil, fmt.Errorf("only COUNT() is supported on joined tables")
+	}
+
+	// Extract WHERE clause and separate by parent/child tables.
+	var cond0, cond1 sql2.Expr
+	for _, cond := range sql2.SplitExprTree(stmt.WhereExpr) {
+		tblName, ok := sql2.ExprTableName(cond)
+		if !ok {
+			return nil, fmt.Errorf("cannot filter across multiple tables in an expression")
+		} else if tblName == "" {
+			return nil, fmt.Errorf("expression must reference a table name")
+		} else if tblName != tbl0.TableName() && tblName != tbl1.TableName() {
+			return nil, fmt.Errorf("no such table: %q", tblName)
+		}
+
+		// Match to parent table.
+		if tblName == tbl0.TableName() {
+			if cond0 == nil {
+				cond0 = cond
+			} else {
+				cond0 = &sql2.BinaryExpr{X: cond0, Op: sql2.AND, Y: cond}
+			}
+			continue
+		}
+
+		// Match to child table.
+		if cond1 == nil {
+			cond1 = cond
+		}
+		cond1 = &sql2.BinaryExpr{X: cond1, Op: sql2.AND, Y: cond}
+	}
+
+	// Convert conditions to PQL.
+	pqlCond0, err := p.planExprPQL(ctx, stmt, cond0)
+	if err != nil {
+		return nil, err
+	} else if pqlCond0 == nil {
+		pqlCond0 = &pql.Call{Name: "All"}
+	}
+
+	pqlCond1, err := p.planExprPQL(ctx, stmt, cond1)
+	if err != nil {
+		return nil, err
+	} else if pqlCond1 == nil {
+		pqlCond1 = &pql.Call{
+			Name: "Row",
+			Args: map[string]interface{}{y.Column.Name: &pql.Condition{
+				Op: pql.NEQ,
+			}},
+		}
+	}
+
+	return NewCountNode(p.executor, tbl0.Name.Name,
+		&StmtColumn{
+			Name: stmt.Columns[0].Name(),
+			Type: sql2.DataTypeInt,
+		},
+		&pql.Call{
+			Name: "Count",
+			Children: []*pql.Call{{
+				Name: "Intersect",
+				Children: []*pql.Call{
+					pqlCond0,
+					{
+						Name: "Distinct",
+						Children: []*pql.Call{
+							pqlCond1,
+						},
+						Args: map[string]interface{}{
+							"index": tbl1.Name.Name,
+							"field": y.Column.Name,
+						},
+					},
+				},
+			}},
+		},
+	), nil
+}
+
 func (p *Planner) planNonAggregateSelectStatement(ctx context.Context, stmt *sql2.SelectStatement) (_ StmtNode, err error) {
 	indexName, err := statementTableName(stmt)
 	if err != nil {
@@ -389,6 +550,53 @@ func (p *Planner) checkStatement(stmt sql2.Statement) error {
 }
 
 func (p *Planner) checkSelectStatement(stmt *sql2.SelectStatement) error {
+	if err := p.expandSelectStatementWildcards(stmt); err != nil {
+		return err
+	}
+
+	// Type check expressions in statement.
+	for _, col := range stmt.Columns {
+		if err := p.checkExpr(&col.Expr, stmt); err != nil {
+			return err
+		}
+	}
+
+	if err := p.checkExpr(&stmt.WhereExpr, stmt); err != nil {
+		return err
+	}
+
+	for i := range stmt.GroupByExprs {
+		if err := p.checkExpr(&stmt.GroupByExprs[i], stmt); err != nil {
+			return err
+		}
+	}
+
+	if err := p.checkExpr(&stmt.HavingExpr, stmt); err != nil {
+		return err
+	}
+
+	for _, term := range stmt.OrderingTerms {
+		if err := p.checkExpr(&term.X, stmt); err != nil {
+			return err
+		}
+	}
+
+	if err := p.checkExpr(&stmt.LimitExpr, stmt); err != nil {
+		return err
+	}
+
+	if err := p.checkExpr(&stmt.OffsetExpr, stmt); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (p *Planner) expandSelectStatementWildcards(stmt *sql2.SelectStatement) error {
+	if !stmt.HasWildcard() {
+		return nil
+	}
+
 	indexName, err := statementTableName(stmt)
 	if err != nil {
 		return err
@@ -441,44 +649,8 @@ func (p *Planner) checkSelectStatement(stmt *sql2.SelectStatement) error {
 	}
 	stmt.Columns = columns
 
-	// Type check expressions in statement.
-	for _, col := range stmt.Columns {
-		if err := p.checkExpr(&col.Expr, stmt); err != nil {
-			return err
-		}
-	}
-
-	if err := p.checkExpr(&stmt.WhereExpr, stmt); err != nil {
-		return err
-	}
-
-	for i := range stmt.GroupByExprs {
-		if err := p.checkExpr(&stmt.GroupByExprs[i], stmt); err != nil {
-			return err
-		}
-	}
-
-	if err := p.checkExpr(&stmt.HavingExpr, stmt); err != nil {
-		return err
-	}
-
-	for _, term := range stmt.OrderingTerms {
-		if err := p.checkExpr(&term.X, stmt); err != nil {
-			return err
-		}
-	}
-
-	if err := p.checkExpr(&stmt.LimitExpr, stmt); err != nil {
-		return err
-	}
-
-	if err := p.checkExpr(&stmt.OffsetExpr, stmt); err != nil {
-		return err
-	}
-
 	return nil
 }
-
 func (p *Planner) checkExpr(expr *sql2.Expr, stmt sql2.Statement) error {
 	if e, err := sql2.Walk(&sqlExprTypeChecker{
 		holder: p.executor.Holder,
@@ -959,20 +1131,17 @@ type CountNode struct {
 	executor  *executor
 	indexName string
 	column    *StmtColumn
-	cond      *pql.Call // conditional
+	call      *pql.Call
 
 	row []interface{}
 }
 
-func NewCountNode(executor *executor, indexName string, column *StmtColumn, cond *pql.Call) *CountNode {
-	if cond == nil {
-		cond = &pql.Call{Name: "All"}
-	}
+func NewCountNode(executor *executor, indexName string, column *StmtColumn, call *pql.Call) *CountNode {
 	return &CountNode{
 		executor:  executor,
 		indexName: indexName,
 		column:    column,
-		cond:      cond,
+		call:      call,
 	}
 }
 
@@ -990,13 +1159,7 @@ func (n *CountNode) Next(ctx context.Context) error {
 		return sql.ErrNoRows
 	}
 
-	q := &pql.Query{
-		Calls: []*pql.Call{
-			{Name: "Count", Children: []*pql.Call{n.cond}},
-		},
-	}
-
-	result, err := n.executor.Execute(ctx, n.indexName, q, nil, nil)
+	result, err := n.executor.Execute(ctx, n.indexName, &pql.Query{Calls: []*pql.Call{n.call}}, nil, nil)
 	if err != nil {
 		return err
 	}
diff --git a/planner_test.go b/planner_test.go
index 68d7505e4..8694b724c 100644
--- a/planner_test.go
+++ b/planner_test.go
@@ -428,6 +428,89 @@ func TestPlanner_GroupBy(t *testing.T) {
 	})
 }
 
+func TestPlanner_InnerJoin(t *testing.T) {
+	c := test.MustRunCluster(t, 1)
+	defer c.Close()
+
+	i0, err := c.GetHolder(0).CreateIndex("i0", pilosa.IndexOptions{TrackExistence: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer i0.Close()
+
+	if _, err := i0.CreateField("a", pilosa.OptFieldTypeInt(0, 1000)); err != nil {
+		t.Fatal(err)
+	}
+
+	i1, err := c.GetHolder(0).CreateIndex("i1", pilosa.IndexOptions{TrackExistence: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer i1.Close()
+
+	if _, err := i1.CreateField("parentid", pilosa.OptFieldTypeInt(0, 1000)); err != nil {
+		t.Fatal(err)
+	} else if _, err := i1.CreateField("x", pilosa.OptFieldTypeInt(0, 1000)); err != nil {
+		t.Fatal(err)
+	}
+
+	// Populate with data.
+	if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{
+		Index: "i0",
+		Query: `
+			Set(1, a=10)
+			Set(2, a=20)
+			Set(3, a=30)
+	`}); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{
+		Index: "i1",
+		Query: `
+			Set(1, parentid=1)
+			Set(1, x=100)
+			
+			Set(2, parentid=1)
+			Set(2, x=200)
+			
+			Set(3, parentid=2)
+			Set(3, x=300)
+	`}); err != nil {
+		t.Fatal(err)
+	}
+
+	t.Run("Count", func(t *testing.T) {
+		results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT COUNT(*) FROM i0 INNER JOIN i1 ON i0._id = i1.parentid`)
+		if diff := cmp.Diff([][]interface{}{
+			{int64(2)},
+		}, results); diff != "" {
+			t.Fatal(diff)
+		}
+
+		if diff := cmp.Diff([]*pilosa.StmtColumn{
+			{Name: "count", Type: "INT"},
+		}, columns); diff != "" {
+			t.Fatal(diff)
+		}
+	})
+
+	t.Run("CountWithParentCondition", func(t *testing.T) {
+		results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT COUNT(*) FROM i0 INNER JOIN i1 ON i0._id = i1.parentid WHERE i0.a = 10`)
+		if diff := cmp.Diff([][]interface{}{
+			{int64(1)},
+		}, results); diff != "" {
+			t.Fatal(diff)
+		}
+
+		if diff := cmp.Diff([]*pilosa.StmtColumn{
+			{Name: "count", Type: "INT"},
+		}, columns); diff != "" {
+			t.Fatal(diff)
+		}
+	})
+}
+
 func mustQueryRows(tb testing.TB, svr *pilosa.Server, q string) (results [][]interface{}, columns []*pilosa.StmtColumn) {
 	tb.Helper()
 
diff --git a/sql2/ast.go b/sql2/ast.go
index 929d1f945..9d2e1a4db 100644
--- a/sql2/ast.go
+++ b/sql2/ast.go
@@ -351,6 +351,119 @@ func ExprString(expr Expr) string {
 	return expr.String()
 }
 
+// ExprTableName returns the name of the table referenced in an expression.
+// Returns ok as false if more than one table referenced. Returns a blank string
+// if no tables are referenced.
+func ExprTableName(expr Expr) (table string, ok bool) {
+	switch expr := expr.(type) {
+	case *BindExpr, *BlobLit, *BoolLit, *Ident, *NullLit, *NumberLit, *StringLit:
+		return "", true
+
+	case *BinaryExpr:
+		x, ok := ExprTableName(expr.X)
+		if !ok {
+			return "", false
+		}
+
+		y, ok := ExprTableName(expr.Y)
+		if !ok {
+			return "", false
+		}
+
+		if x == "" {
+			return y, true
+		} else if y == "" {
+			return x, true
+		} else if x == y {
+			return x, true
+		}
+		return "", false
+
+	case *Call:
+		for _, arg := range expr.Args {
+			tbl, ok := ExprTableName(arg)
+			if !ok || (table != "" && tbl != table) {
+				return "", false
+			}
+			table = tbl
+		}
+		return table, true
+
+	case *CaseExpr:
+		tbl, ok := ExprTableName(expr.Operand)
+		if !ok || (table != "" && tbl != table) {
+			return "", false
+		}
+		table = tbl
+
+		tbl, ok = ExprTableName(expr.ElseExpr)
+		if !ok || (table != "" && tbl != table) {
+			return "", false
+		}
+		table = tbl
+
+		for _, blk := range expr.Blocks {
+			tbl, ok := ExprTableName(blk.Condition)
+			if !ok || (table != "" && tbl != table) {
+				return "", false
+			}
+			table = tbl
+
+			tbl, ok = ExprTableName(blk.Body)
+			if !ok || (table != "" && tbl != table) {
+				return "", false
+			}
+			table = tbl
+		}
+		return table, true
+
+	case *CastExpr:
+		return ExprTableName(expr.X)
+
+	case *Exists:
+		return "", false // TODO
+
+	case *ExprList:
+		for _, e := range expr.Exprs {
+			tbl, ok := ExprTableName(e)
+			if !ok || (table != "" && tbl != table) {
+				return "", false
+			}
+			table = tbl
+		}
+		return table, true
+
+	case *ParenExpr:
+		return ExprTableName(expr.X)
+
+	case *QualifiedRef:
+		return expr.Table.Name, true
+
+	case *Raise:
+		return "", true
+
+	case *Range:
+		tbl, ok := ExprTableName(expr.X)
+		if !ok || (table != "" && tbl != table) {
+			return "", false
+		}
+		table = tbl
+
+		tbl, ok = ExprTableName(expr.Y)
+		if !ok || (table != "" && tbl != table) {
+			return "", false
+		}
+		table = tbl
+		return table, true
+
+	case *UnaryExpr:
+		return ExprTableName(expr.X)
+
+	default:
+		return "", false
+	}
+}
+
 // SplitExprTree splits apart expr so it is a list of all AND joined expressions.
 // For example, the expression "A AND B AND (C OR (D AND E))" would be split into
 // a list of "A", "B", "C OR (D AND E)".
@@ -3001,6 +3114,23 @@ func (s *SelectStatement) IsAggregate() bool {
 	return false
 }
 
+// HasWildcard returns true any result column contains a wildcard (STAR).
+func (s *SelectStatement) HasWildcard() bool {
+	for _, col := range s.Columns {
+		// Unqualified wildcard.
+		if col.Star.IsValid() {
+			return true
+		}
+
+		// Table-qualified wildcard.
+		if ref, ok := col.Expr.(*QualifiedRef); ok && ref.Star.IsValid() {
+			return true
+		}
+	}
+
+	return false
+}
+
 // String returns the string representation of the statement.
 func (s *SelectStatement) String() string {
 	var buf bytes.Buffer

From 786bebe58bf7e86e3cf75e4c400edf3b691382f8 Mon Sep 17 00:00:00 2001
From: Todd Gruben <todd@molecula.com>
Date: Fri, 8 Oct 2021 10:49:41 -0500
Subject: [PATCH 65/66] partial backup/restore

---
 cmd/backup.go        |  1 +
 ctl/backup.go        | 18 +++++++++++++++-
 ctl/restore.go       | 50 +++++++++++++++++++++++++++++++++++++-------
 testBackupRestore.sh | 22 +++++++++++++++++++
 4 files changed, 83 insertions(+), 8 deletions(-)

diff --git a/cmd/backup.go b/cmd/backup.go
index 68baeb014..d81029d7a 100644
--- a/cmd/backup.go
+++ b/cmd/backup.go
@@ -40,6 +40,7 @@ Backs up a FeatureBase server to a local, tar-formatted snapshot file.
 	flags.BoolVar(&cmd.NoSync, "no-sync", false, "disable file sync")
 	flags.IntVar(&cmd.Concurrency, "concurrency", cmd.Concurrency, "number of concurrent backup goroutines")
 	flags.StringVar(&cmd.Host, "host", "localhost:10101", "host:port of FeatureBase.")
+	flags.StringVar(&cmd.Index, "index", "", "index to backup, default backs up all indexes. ")
 	ctl.SetTLSConfig(flags, "", &cmd.TLS.CertificatePath, &cmd.TLS.CertificateKeyPath, &cmd.TLS.CACertPath, &cmd.TLS.SkipVerify, &cmd.TLS.EnableClientVerification)
 	return ccmd
 }
diff --git a/ctl/backup.go b/ctl/backup.go
index e68ce52d2..cdcfb1427 100644
--- a/ctl/backup.go
+++ b/ctl/backup.go
@@ -24,7 +24,7 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/molecula/featurebase/v2"
+	pilosa "github.com/molecula/featurebase/v2"
 	"github.com/molecula/featurebase/v2/http"
 	"github.com/molecula/featurebase/v2/server"
 	"github.com/molecula/featurebase/v2/topology"
@@ -38,6 +38,9 @@ type BackupCommand struct { // nolint: maligned
 	// Destination host and port.
 	Host string `json:"host"`
 
+	// Optional Index filter
+	Index string `json:"index"`
+
 	// Path to write the backup to.
 	OutputDir string
 
@@ -91,6 +94,19 @@ func (cmd *BackupCommand) Run(ctx context.Context) (err error) {
 	if err != nil {
 		return fmt.Errorf("getting schema: %w", err)
 	}
+	if cmd.Index != "" {
+		for _, idx := range indexes {
+			if idx.Name == cmd.Index {
+				indexes = make([]*pilosa.IndexInfo, 0)
+				indexes = append(indexes, idx)
+				break
+			}
+		}
+		if len(indexes) <= 0 {
+			return fmt.Errorf("Index not found to back up")
+		}
+	}
+
 	schema := &pilosa.Schema{Indexes: indexes}
 
 	// Ensure output directory doesn't exist; then create output directory.
diff --git a/ctl/restore.go b/ctl/restore.go
index 26d72c788..cf03b9eab 100644
--- a/ctl/restore.go
+++ b/ctl/restore.go
@@ -17,6 +17,7 @@ package ctl
 import (
 	"context"
 	"crypto/tls"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -26,9 +27,10 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/molecula/featurebase/v2"
+	pilosa "github.com/molecula/featurebase/v2"
 	"github.com/molecula/featurebase/v2/server"
 	"github.com/molecula/featurebase/v2/topology"
+	"github.com/molecula/featurebase/v2/vprint"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -102,7 +104,6 @@ func (cmd *RestoreCommand) Run(ctx context.Context) (err error) {
 	} else if err := cmd.restoreIDAlloc(ctx, primary); err != nil {
 		return fmt.Errorf("cannot restore idalloc: %w", err)
 	}
-
 	if err := cmd.restoreShards(ctx); err != nil {
 		return fmt.Errorf("cannot restore shards: %w", err)
 	} else if err := cmd.restoreIndexTranslation(ctx); err != nil {
@@ -128,11 +129,46 @@ func (cmd *RestoreCommand) restoreSchema(ctx context.Context, primary *topology.
 	}
 	defer f.Close()
 
-	cmd.Logger().Printf("Load Schema")
-	url := primary.URI.Path("/schema")
-
-	var client http.Client
-	_, err = client.Post(url, "application/json", f)
+	existingSchema, err := cmd.client.Schema(ctx)
+	if len(existingSchema) == 0 {
+		cmd.Logger().Printf("Load Schema")
+		url := primary.URI.Path("/schema")
+		var client http.Client
+		_, err = client.Post(url, "application/json", f)
+	} else {
+		schema := &pilosa.Schema{}
+		if err := json.NewDecoder(f).Decode(schema); err != nil {
+			if err != nil {
+				return err
+			}
+		}
+		exists := func(indexName string) bool {
+			for _, i := range existingSchema {
+				if i.Name == indexName {
+					return true
+				}
+			}
+			return false
+		}
+		//NOTE SHOULD ONLY BE ONE
+		for _, index := range schema.Indexes {
+			if exists(index.Name) {
+				return errors.New(fmt.Sprintf("Index Exists %v", index.Name))
+			}
+			vprint.VV("Create INDEX %v", index.Name)
+			err = cmd.client.CreateIndex(ctx, index.Name, index.Options)
+			if err != nil {
+				return err
+			}
+			for _, field := range index.Fields {
+				vprint.VV("Create Field %v", field.Name)
+				err = cmd.client.CreateFieldWithOptions(ctx, index.Name, field.Name, field.Options)
+				if err != nil {
+					return err
+				}
+			}
+		}
+	}
 	return err
 }
 
diff --git a/testBackupRestore.sh b/testBackupRestore.sh
index 6d0dbc67f..7bcbe3894 100755
--- a/testBackupRestore.sh
+++ b/testBackupRestore.sh
@@ -33,3 +33,25 @@ else
 	echo "FAIL Single"
 	exit 1
 fi
+
+datagen --source texas_health -e 9999 --pilosa.index newsink --pilosa.batch-size 10000 --pilosa.hosts pilosa0:10101
+before=$(/featurebase chksum --host pilosa0:10101)
+/featurebase backup -o newbackupdir --host pilosa0:10101 --index newsink
+curl -X DELETE -s pilosa0:10101/index/newsink
+/featurebase restore -s newbackupdir --host pilosa0:10101
+after=$(/featurebase chksum --host pilosa0:10101)
+if [ "$before" = "$after" ]; then
+	echo "PASS Cluster Table"
+else
+	echo "FAIL Single Table"
+	exit 1
+fi
+/featurebase restore -s newbackupdir --host pilosax:10101
+single=$(/featurebase chksum --host pilosax:10101)
+if [ "$before" = "$single" ]; then
+	echo "PASS Single Table"
+	exit 0
+else
+	echo "FAIL Single Table"
+	exit 1
+fi

From 71f1e6f1dd9e1a39d58549d0b0a994fbfa74144b Mon Sep 17 00:00:00 2001
From: Todd Gruben <todd@molecula.com>
Date: Fri, 8 Oct 2021 12:20:53 -0500
Subject: [PATCH 66/66] linter

---
 ctl/restore.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/ctl/restore.go b/ctl/restore.go
index cf03b9eab..87d4d3c87 100644
--- a/ctl/restore.go
+++ b/ctl/restore.go
@@ -30,7 +30,6 @@ import (
 	pilosa "github.com/molecula/featurebase/v2"
 	"github.com/molecula/featurebase/v2/server"
 	"github.com/molecula/featurebase/v2/topology"
-	"github.com/molecula/featurebase/v2/vprint"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -150,18 +149,19 @@ func (cmd *RestoreCommand) restoreSchema(ctx context.Context, primary *topology.
 			}
 			return false
 		}
+		logger := cmd.Logger()
 		//NOTE SHOULD ONLY BE ONE
 		for _, index := range schema.Indexes {
 			if exists(index.Name) {
-				return errors.New(fmt.Sprintf("Index Exists %v", index.Name))
+				return fmt.Errorf("Index Exists %v", index.Name)
 			}
-			vprint.VV("Create INDEX %v", index.Name)
+			logger.Printf("Create INDEX %v", index.Name)
 			err = cmd.client.CreateIndex(ctx, index.Name, index.Options)
 			if err != nil {
 				return err
 			}
 			for _, field := range index.Fields {
-				vprint.VV("Create Field %v", field.Name)
+				logger.Printf("Create Field %v", field.Name)
 				err = cmd.client.CreateFieldWithOptions(ctx, index.Name, field.Name, field.Options)
 				if err != nil {
 					return err