From 3781b6e7eb40352b7a2defe6a4574800cd1cc743 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Tue, 3 Jul 2018 11:05:33 -0500
Subject: [PATCH 01/32] remove redundant calls to SetupServer

---
 test/pilosa.go | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/test/pilosa.go b/test/pilosa.go
index 4774b4cf8..a40f2e956 100644
--- a/test/pilosa.go
+++ b/test/pilosa.go
@@ -63,11 +63,6 @@ func NewCommand(opts ...server.CommandOption) *Command {
 	m.Command.Stdout = &m.Stdout
 	m.Command.Stderr = &m.Stderr
 
-	err = m.SetupServer()
-	if err != nil {
-		panic(err)
-	}
-
 	if testing.Verbose() {
 		m.Command.Stdout = io.MultiWriter(os.Stdout, m.Command.Stdout)
 		m.Command.Stderr = io.MultiWriter(os.Stderr, m.Command.Stderr)
@@ -117,10 +112,6 @@ func (m *Command) Reopen() error {
 	config := m.Command.Config
 	m.Command = server.NewCommand(os.Stdin, os.Stdout, os.Stderr, m.commandOptions...)
 	m.Command.Config = config
-	err := m.SetupServer()
-	if err != nil {
-		return errors.Wrap(err, "setting up server")
-	}
 
 	// Run new program.
 	if err := m.Start(); err != nil {

From f6aef32093a747527b2aa1d8cf3510dbb40d903e Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Tue, 3 Jul 2018 12:58:18 -0500
Subject: [PATCH 02/32] make gossip's interface to Pilosa the API struct rather
 than effectively being pilosa.Server

---
 api.go           | 23 ++++++++--------
 broadcast.go     |  6 ++++-
 field.go         | 15 +++++++++++
 gossip/gossip.go | 68 ++++++++++++++++++++++--------------------------
 holder.go        |  2 +-
 http/handler.go  |  8 +++---
 index.go         | 21 +++++++++++++--
 server.go        |  2 ++
 server/server.go |  2 +-
 9 files changed, 88 insertions(+), 59 deletions(-)

diff --git a/api.go b/api.go
index 33fff083d..3fca21cab 100644
--- a/api.go
+++ b/api.go
@@ -331,8 +331,8 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin
 	}
 
 	// Validate that this handler owns the shard.
-	if !api.cluster.ownsShard(api.LocalID(), indexName, shard) {
-		api.server.logger.Printf("node %s does not own shard %d of index %s", api.LocalID(), shard, indexName)
+	if !api.cluster.ownsShard(api.Node().ID, indexName, shard) {
+		api.server.logger.Printf("node %s does not own shard %d of index %s", api.Node().ID, shard, indexName)
 		return ErrClusterDoesNotOwnShard
 	}
 
@@ -477,6 +477,10 @@ func (api *API) Hosts(ctx context.Context) []*Node {
 	return api.cluster.Nodes
 }
 
+func (api *API) Node() *Node {
+	return api.server.Node()
+}
+
 // RecalculateCaches forces all TopN caches to be updated. Used mainly for integration tests.
 func (api *API) RecalculateCaches(ctx context.Context) error {
 	if err := api.validate(apiRecalculateCaches); err != nil {
@@ -517,15 +521,10 @@ func (api *API) ClusterMessage(ctx context.Context, reqBody io.Reader) error {
 	return nil
 }
 
-// LocalID returns the current node's ID.
-func (api *API) LocalID() string {
-	return api.cluster.Node.ID
-}
-
 // Schema returns information about each index in Pilosa including which fields
 // and views they contain.
-func (api *API) Schema(ctx context.Context) []*IndexInfo {
-	return api.holder.Schema()
+func (api *API) Schema(ctx context.Context) []*Index {
+	return api.holder.Indexes()
 }
 
 // Views returns the views in the given field.
@@ -720,8 +719,8 @@ func (api *API) LongQueryTime() time.Duration {
 
 func (api *API) indexField(indexName string, fieldName string, shard uint64) (*Index, *Field, error) {
 	// Validate that this handler owns the shard.
-	if !api.cluster.ownsShard(api.LocalID(), indexName, shard) {
-		api.server.logger.Printf("node %s does not own shard %d of index %s", api.LocalID(), shard, indexName)
+	if !api.cluster.ownsShard(api.Node().ID, indexName, shard) {
+		api.server.logger.Printf("node %s does not own shard %d of index %s", api.Node().ID, shard, indexName)
 		return nil, nil, ErrClusterDoesNotOwnShard
 	}
 
@@ -755,7 +754,7 @@ func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode
 	}
 
 	// If the new coordinator is this node, do the SetCoordinator directly.
-	if newNode.ID == api.LocalID() {
+	if newNode.ID == api.Node().ID {
 		return oldNode, newNode, api.cluster.setCoordinator(newNode)
 	}
 
diff --git a/broadcast.go b/broadcast.go
index 37452292a..19e927c18 100644
--- a/broadcast.go
+++ b/broadcast.go
@@ -65,6 +65,7 @@ const (
 	messageTypeNodeState
 	messageTypeRecalculateCaches
 	messageTypeNodeEvent
+	messageTypeNodeStatus
 )
 
 // MarshalMessage encodes the protobuf message into a byte slice.
@@ -101,6 +102,8 @@ func MarshalMessage(m proto.Message) ([]byte, error) {
 		typ = messageTypeRecalculateCaches
 	case *internal.NodeEventMessage:
 		typ = messageTypeNodeEvent
+	case *internal.NodeStatus:
+		typ = messageTypeNodeStatus
 	default:
 		return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj))
 	}
@@ -114,7 +117,6 @@ func MarshalMessage(m proto.Message) ([]byte, error) {
 // UnmarshalMessage decodes the byte slice into a protobuf message.
 func UnmarshalMessage(buf []byte) (proto.Message, error) {
 	typ, buf := buf[0], buf[1:]
-
 	var m proto.Message
 	switch typ {
 	case messageTypeCreateShard:
@@ -147,6 +149,8 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) {
 		m = &internal.RecalculateCaches{}
 	case messageTypeNodeEvent:
 		m = &internal.NodeEventMessage{}
+	case messageTypeNodeStatus:
+		m = &internal.NodeStatus{}
 	default:
 		return nil, fmt.Errorf("invalid message type: %d", typ)
 	}
diff --git a/field.go b/field.go
index 2e4897f91..76be9cf3f 100644
--- a/field.go
+++ b/field.go
@@ -1073,6 +1073,21 @@ func (f *Field) ImportValue(columnIDs []uint64, values []int64) error {
 	return nil
 }
 
+func (f *Field) MarshalJSON() ([]byte, error) {
+	thing := struct {
+		Name    string
+		Options FieldOptions
+		Views   []*viewInfo
+	}{
+		Name:    f.Name(),
+		Options: f.Options(),
+	}
+	for _, viewname := range f.viewNames() {
+		thing.Views = append(thing.Views, &viewInfo{Name: viewname})
+	}
+	return json.Marshal(thing)
+}
+
 // encodeFields converts a into its internal representation.
 func encodeFields(a []*Field) []*internal.Field {
 	other := make([]*internal.Field, len(a))
diff --git a/gossip/gossip.go b/gossip/gossip.go
index 4f562af5e..3dc7c202b 100644
--- a/gossip/gossip.go
+++ b/gossip/gossip.go
@@ -15,6 +15,8 @@
 package gossip
 
 import (
+	"bytes"
+	"context"
 	"fmt"
 	"io/ioutil"
 	"log"
@@ -42,8 +44,8 @@ type GossipMemberSet struct {
 
 	broadcasts *memberlist.TransmitLimitedQueue
 
-	pserver pilosa.MemberServer
-	config  *gossipConfig
+	papi   *pilosa.API
+	config *gossipConfig
 
 	Logger pilosa.Logger
 
@@ -145,9 +147,10 @@ func WithLogger(logger *log.Logger) GossipMemberSetOption {
 }
 
 // NewGossipMemberSet returns a new instance of GossipMemberSet based on options.
-func NewGossipMemberSet(cfg Config, s *pilosa.Server, options ...GossipMemberSetOption) (*GossipMemberSet, error) {
-	host := s.Node().URI.Host()
+func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...GossipMemberSetOption) (*GossipMemberSet, error) {
+	host := api.Node().URI.Host()
 	g := &GossipMemberSet{
+		papi:   api,
 		Logger: pilosa.NopLogger,
 	}
 
@@ -157,7 +160,7 @@ func NewGossipMemberSet(cfg Config, s *pilosa.Server, options ...GossipMemberSet
 			return nil, errors.Wrap(err, "executing option")
 		}
 	}
-	ger := newGossipEventReceiver(g.logger, s)
+	ger := newGossipEventReceiver(g.logger, api)
 	g.gossipEventReceiver = ger
 
 	if g.transport == nil {
@@ -189,11 +192,11 @@ func NewGossipMemberSet(cfg Config, s *pilosa.Server, options ...GossipMemberSet
 	// memberlist config
 	conf := memberlist.DefaultWANConfig()
 	conf.Transport = g.transport.Net
-	conf.Name = s.Node().ID
-	conf.BindAddr = s.Node().URI.Host()
+	conf.Name = api.Node().ID
+	conf.BindAddr = api.Node().URI.Host()
 	conf.BindPort = port
 	conf.AdvertisePort = port
-	conf.AdvertiseAddr = hostToIP(s.Node().URI.Host())
+	conf.AdvertiseAddr = hostToIP(api.Node().URI.Host())
 	//
 	conf.TCPTimeout = time.Duration(cfg.StreamTimeout)
 	conf.SuspicionMult = cfg.SuspicionMult
@@ -214,14 +217,12 @@ func NewGossipMemberSet(cfg Config, s *pilosa.Server, options ...GossipMemberSet
 		gossipSeeds:      cfg.Seeds,
 	}
 
-	g.pserver = s
-
 	return g, nil
 }
 
 // NodeMeta implementation of the memberlist.Delegate interface.
 func (g *GossipMemberSet) NodeMeta(limit int) []byte {
-	buf, err := proto.Marshal(pilosa.EncodeNode(g.pserver.Node()))
+	buf, err := proto.Marshal(pilosa.EncodeNode(g.papi.Node()))
 	if err != nil {
 		g.Logger.Printf("marshal message error: %s", err)
 		return []byte{}
@@ -232,14 +233,9 @@ func (g *GossipMemberSet) NodeMeta(limit int) []byte {
 // NotifyMsg implementation of the memberlist.Delegate interface
 // called when a user-data message is received.
 func (g *GossipMemberSet) NotifyMsg(b []byte) {
-	m, err := pilosa.UnmarshalMessage(b)
+	err := g.papi.ClusterMessage(context.Background(), bytes.NewBuffer(b))
 	if err != nil {
-		g.Logger.Printf("unmarshal message error: %s", err)
-		return
-	}
-	if err := g.pserver.ReceiveMessage(m); err != nil {
-		g.Logger.Printf("receive message error: %s", err)
-		return
+		g.Logger.Printf("cluster message error: %s", err)
 	}
 }
 
@@ -252,14 +248,14 @@ func (g *GossipMemberSet) GetBroadcasts(overhead, limit int) [][]byte {
 // LocalState implementation of the memberlist.Delegate interface
 // sends this Node's state data.
 func (g *GossipMemberSet) LocalState(join bool) []byte {
-	pb, err := g.pserver.LocalStatus()
-	if err != nil {
-		g.Logger.Printf("error getting local state, err=%s", err)
-		return []byte{}
+	pb := &internal.NodeStatus{
+		Node:      pilosa.EncodeNode(g.papi.Node()),
+		MaxShards: &internal.MaxShards{Standard: g.papi.MaxShards(context.Background())},
+		Schema:    &internal.Schema{Indexes: pilosa.EncodeIndexes(g.papi.Schema(context.Background()))},
 	}
 
 	// Marshal nodestate data to bytes.
-	buf, err := proto.Marshal(pb)
+	buf, err := pilosa.MarshalMessage(pb)
 	if err != nil {
 		g.Logger.Printf("error marshalling nodestate data, err=%s", err)
 		return []byte{}
@@ -270,13 +266,7 @@ func (g *GossipMemberSet) LocalState(join bool) []byte {
 // MergeRemoteState implementation of the memberlist.Delegate interface
 // receive and process the remote side's LocalState.
 func (g *GossipMemberSet) MergeRemoteState(buf []byte, join bool) {
-	// Unmarshal nodestate data.
-	var pb internal.NodeStatus
-	if err := proto.Unmarshal(buf, &pb); err != nil {
-		g.Logger.Printf("error unmarshalling nodestate data, err=%s", err)
-		return
-	}
-	err := g.pserver.HandleRemoteStatus(&pb)
+	err := g.papi.ClusterMessage(context.Background(), bytes.NewBuffer(buf))
 	if err != nil {
 		g.Logger.Printf("merge state error: %s", err)
 	}
@@ -288,18 +278,18 @@ func (g *GossipMemberSet) MergeRemoteState(buf []byte, join bool) {
 // Care must be taken that events are processed in a timely manner from
 // the channel, since this delegate will block until an event can be sent.
 type gossipEventReceiver struct {
-	ch           chan memberlist.NodeEvent
-	eventHandler *pilosa.Server
+	ch   chan memberlist.NodeEvent
+	papi *pilosa.API
 
 	logger *log.Logger
 }
 
 // newGossipEventReceiver returns a new instance of GossipEventReceiver.
-func newGossipEventReceiver(logger *log.Logger, pserver *pilosa.Server) *gossipEventReceiver {
+func newGossipEventReceiver(logger *log.Logger, papi *pilosa.API) *gossipEventReceiver {
 	ger := &gossipEventReceiver{
-		ch:           make(chan memberlist.NodeEvent, 1),
-		logger:       logger,
-		eventHandler: pserver,
+		ch:     make(chan memberlist.NodeEvent, 1),
+		logger: logger,
+		papi:   papi,
 	}
 	go ger.listen()
 	return ger
@@ -342,7 +332,11 @@ func (g *gossipEventReceiver) listen() {
 			Event: uint32(nodeEventType),
 			Node:  &n,
 		}
-		if err := g.eventHandler.ReceiveMessage(ne); err != nil {
+		buf, err := pilosa.MarshalMessage(ne)
+		if err != nil {
+			panic(err)
+		}
+		if err := g.papi.ClusterMessage(context.Background(), bytes.NewBuffer(buf)); err != nil {
 			g.logger.Printf("receive event error: %s", err)
 		}
 	}
diff --git a/holder.go b/holder.go
index bc255abba..e599d21f2 100644
--- a/holder.go
+++ b/holder.go
@@ -267,7 +267,7 @@ func (h *Holder) encodeMaxShards() *internal.MaxShards {
 // encodeSchema creates an internal representation of schema.
 func (h *Holder) encodeSchema() *internal.Schema {
 	return &internal.Schema{
-		Indexes: encodeIndexes(h.Indexes()),
+		Indexes: EncodeIndexes(h.Indexes()),
 	}
 }
 
diff --git a/http/handler.go b/http/handler.go
index 9d6b9e080..9195f05e9 100644
--- a/http/handler.go
+++ b/http/handler.go
@@ -358,9 +358,7 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) {
 	}
 
 	schema := h.API.Schema(r.Context())
-	if err := json.NewEncoder(w).Encode(getSchemaResponse{
-		Indexes: schema,
-	}); err != nil {
+	if err := json.NewEncoder(w).Encode(map[string]interface{}{"indexes": schema}); err != nil {
 		h.Logger.Printf("write schema response error: %s", err)
 	}
 }
@@ -374,7 +372,7 @@ func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) {
 	status := getStatusResponse{
 		State:   h.API.State(),
 		Nodes:   h.API.Hosts(r.Context()),
-		LocalID: h.API.LocalID(),
+		LocalID: h.API.Node().ID,
 	}
 	if err := json.NewEncoder(w).Encode(status); err != nil {
 		h.Logger.Printf("write status response error: %s", err)
@@ -467,7 +465,7 @@ func (h *Handler) handleGetIndex(w http.ResponseWriter, r *http.Request) {
 	}
 	indexName := mux.Vars(r)["index"]
 	for _, idx := range h.API.Schema(r.Context()) {
-		if idx.Name == indexName {
+		if idx.Name() == indexName {
 			if err := json.NewEncoder(w).Encode(idx); err != nil {
 				h.Logger.Printf("write response error: %s", err)
 			}
diff --git a/index.go b/index.go
index d1a6a8ee5..98f50eced 100644
--- a/index.go
+++ b/index.go
@@ -15,6 +15,7 @@
 package pilosa
 
 import (
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -75,6 +76,22 @@ func NewIndex(path, name string) (*Index, error) {
 	}, nil
 }
 
+func (i *Index) MarshalJSON() ([]byte, error) {
+	fields := make([]*Field, 0, len(i.fields))
+	for _, f := range i.fields {
+
+		fields = append(fields, f)
+	}
+	thing := struct {
+		Name   string
+		Fields []*Field
+	}{
+		Name:   i.name,
+		Fields: fields,
+	}
+	return json.Marshal(thing)
+}
+
 // Name returns name of the index.
 func (i *Index) Name() string { return i.name }
 
@@ -386,8 +403,8 @@ func (p indexInfoSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
 func (p indexInfoSlice) Len() int           { return len(p) }
 func (p indexInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
 
-// encodeIndexes converts a into its internal representation.
-func encodeIndexes(a []*Index) []*internal.Index {
+// EncodeIndexes converts a into its internal representation.
+func EncodeIndexes(a []*Index) []*internal.Index {
 	other := make([]*internal.Index, len(a))
 	for i := range a {
 		other[i] = encodeIndex(a[i])
diff --git a/server.go b/server.go
index 7fbd94066..14f2be179 100644
--- a/server.go
+++ b/server.go
@@ -511,6 +511,8 @@ func (s *Server) ReceiveMessage(pb proto.Message) error {
 		s.holder.RecalculateCaches()
 	case *internal.NodeEventMessage:
 		s.cluster.ReceiveEvent(DecodeNodeEvent(obj))
+	case *internal.NodeStatus:
+		s.HandleRemoteStatus(pb)
 	}
 
 	return nil
diff --git a/server/server.go b/server/server.go
index 164dc94e3..7781ed4bc 100644
--- a/server/server.go
+++ b/server/server.go
@@ -317,7 +317,7 @@ func (m *Command) SetupNetworking() error {
 
 	gossipMemberSet, err := gossip.NewGossipMemberSet(
 		m.Config.Gossip,
-		m.Server,
+		m.API,
 		gossip.WithLogger(m.logger.Logger()),
 		gossip.WithTransport(m.gossipTransport),
 	)

From 5ff77c816a3f956cf59103b317be9ed4870d41ef Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Tue, 3 Jul 2018 13:26:51 -0500
Subject: [PATCH 03/32] get rid of unecessary server stuff and export node and
 uri

---
 api.go                  |  6 ++--
 ctl/export_test.go      |  2 +-
 ctl/import_test.go      |  8 ++---
 http/translator_test.go |  6 ++--
 server.go               | 69 ++++++++++-------------------------------
 server/server.go        |  4 +--
 server/server_test.go   |  2 +-
 test/pilosa.go          |  4 +--
 8 files changed, 33 insertions(+), 68 deletions(-)

diff --git a/api.go b/api.go
index 3fca21cab..abf2a84ea 100644
--- a/api.go
+++ b/api.go
@@ -477,8 +477,10 @@ func (api *API) Hosts(ctx context.Context) []*Node {
 	return api.cluster.Nodes
 }
 
+// Node gets the ID, URI and coordinator status for this particular node.
 func (api *API) Node() *Node {
-	return api.server.Node()
+	node := api.server.node()
+	return &node
 }
 
 // RecalculateCaches forces all TopN caches to be updated. Used mainly for integration tests.
@@ -515,7 +517,7 @@ func (api *API) ClusterMessage(ctx context.Context, reqBody io.Reader) error {
 	}
 
 	// Forward the error message.
-	if err := api.server.ReceiveMessage(pb); err != nil {
+	if err := api.server.receiveMessage(pb); err != nil {
 		return errors.Wrap(err, "receiving message")
 	}
 	return nil
diff --git a/ctl/export_test.go b/ctl/export_test.go
index e959da6bc..e3189efe3 100644
--- a/ctl/export_test.go
+++ b/ctl/export_test.go
@@ -49,7 +49,7 @@ func TestExportCommand_Run(t *testing.T) {
 	buf := bytes.Buffer{}
 	stdin, stdout, stderr := GetIO(buf)
 	cm := NewExportCommand(stdin, stdout, stderr)
-	hostport := cmd.Server.URI.HostPort()
+	hostport := cmd.API.Node().URI.HostPort()
 	cm.Host = hostport
 
 	http.DefaultClient.Do(test.MustNewHTTPRequest("POST", "http://"+hostport+"/index/i", strings.NewReader("")))
diff --git a/ctl/import_test.go b/ctl/import_test.go
index 32792c02d..56970e9be 100644
--- a/ctl/import_test.go
+++ b/ctl/import_test.go
@@ -62,7 +62,7 @@ func TestImportCommand_Run(t *testing.T) {
 	}
 
 	cmd := test.MustRunCluster(t, 1)[0]
-	cm.Host = cmd.Server.URI.HostPort()
+	cm.Host = cmd.API.Node().URI.HostPort()
 
 	cm.Index = "i"
 	cm.Field = "f"
@@ -87,7 +87,7 @@ func TestImportCommand_RunValue(t *testing.T) {
 	}
 
 	cmd := test.MustRunCluster(t, 1)[0]
-	cm.Host = cmd.Server.URI.HostPort()
+	cm.Host = cmd.API.Node().URI.HostPort()
 
 	http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader("")))
 	http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max": 100}}`)))
@@ -107,7 +107,7 @@ func TestImportCommand_InvalidFile(t *testing.T) {
 	buf := bytes.Buffer{}
 	stdin, stdout, stderr := GetIO(buf)
 	cm := NewImportCommand(stdin, stdout, stderr)
-	cm.Host = cmd.Server.URI.HostPort()
+	cm.Host = cmd.API.Node().URI.HostPort()
 	cm.Index = "i"
 	cm.Field = "f"
 	file, err := ioutil.TempFile("", "import.csv")
@@ -188,7 +188,7 @@ func TestImportCommand_BugOverwriteValue(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	cm.Host = cmd.Server.URI.HostPort()
+	cm.Host = cmd.API.Node().URI.HostPort()
 
 	http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader("")))
 	http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max":2147483648 }}`)))
diff --git a/http/translator_test.go b/http/translator_test.go
index a3a5e8603..04531dab5 100644
--- a/http/translator_test.go
+++ b/http/translator_test.go
@@ -75,7 +75,7 @@ func TestTranslateStore_Reader(t *testing.T) {
 			defer main.Close()
 
 			// Connect to server and stream all available data.
-			store := http.NewTranslateStore(main.Server.URI.String())
+			store := http.NewTranslateStore(main.URL())
 
 			rc, err := store.Reader(context.Background(), 100)
 			if err != nil {
@@ -128,7 +128,7 @@ func TestTranslateStore_Reader(t *testing.T) {
 
 			// Connect to server and begin streaming.
 			ctx, cancel := context.WithCancel(context.Background())
-			store := http.NewTranslateStore(main.Server.URI.String())
+			store := http.NewTranslateStore(main.URL())
 			if _, err := store.Reader(ctx, 0); err != nil {
 				t.Fatal(err)
 			}
@@ -155,7 +155,7 @@ func TestTranslateStore_Reader(t *testing.T) {
 		main := test.MustRunCluster(t, 1, []server.CommandOption{opts})[0]
 		defer main.Close()
 
-		_, err := http.NewTranslateStore(main.Server.URI.String()).Reader(context.Background(), 0)
+		_, err := http.NewTranslateStore(main.URL()).Reader(context.Background(), 0)
 		if err != pilosa.ErrNotImplemented {
 			t.Fatalf("unexpected error: %s", err)
 		}
diff --git a/server.go b/server.go
index 14f2be179..56195b427 100644
--- a/server.go
+++ b/server.go
@@ -36,12 +36,11 @@ import (
 
 // Default server settings.
 const (
-	DefaultDiagnosticServer = "https://diagnostics.pilosa.com/v0/diagnostics"
+	defaultDiagnosticServer = "https://diagnostics.pilosa.com/v0/diagnostics"
 )
 
 // Ensure Server implements interfaces.
 var _ broadcaster = &Server{}
-var _ MemberServer = &Server{}
 
 // Server represents a holder wrapped by a running HTTP server.
 type Server struct {
@@ -64,7 +63,7 @@ type Server struct {
 	logger     Logger
 
 	nodeID              string
-	URI                 URI
+	uri                 URI
 	antiEntropyInterval time.Duration
 	metricInterval      time.Duration
 	diagnosticInterval  time.Duration
@@ -188,7 +187,7 @@ func OptServerDiagnosticsInterval(dur time.Duration) ServerOption {
 
 func OptServerURI(uri *URI) ServerOption {
 	return func(s *Server) error {
-		s.URI = *uri
+		s.uri = *uri
 		return nil
 	}
 }
@@ -230,7 +229,7 @@ func NewServer(opts ...ServerOption) (*Server, error) {
 		closing:     make(chan struct{}),
 		cluster:     newCluster(),
 		holder:      NewHolder(),
-		diagnostics: NewDiagnosticsCollector(DefaultDiagnosticServer),
+		diagnostics: NewDiagnosticsCollector(defaultDiagnosticServer),
 		systemInfo:  NewNopSystemInfo(),
 
 		gcNotifier: NopGCNotifier,
@@ -277,7 +276,7 @@ func NewServer(opts ...ServerOption) (*Server, error) {
 	// Set Cluster Node.
 	node := &Node{
 		ID:            s.nodeID,
-		URI:           s.URI,
+		URI:           s.uri,
 		IsCoordinator: s.cluster.Coordinator == s.nodeID,
 	}
 	s.cluster.Node = node
@@ -431,8 +430,8 @@ func (s *Server) monitorAntiEntropy() {
 	}
 }
 
-// ReceiveMessage represents an implementation of BroadcastHandler.
-func (s *Server) ReceiveMessage(pb proto.Message) error {
+// receiveMessage represents an implementation of BroadcastHandler.
+func (s *Server) receiveMessage(pb proto.Message) error {
 	switch obj := pb.(type) {
 	case *internal.CreateShardMessage:
 		idx := s.holder.Index(obj.Index)
@@ -512,7 +511,7 @@ func (s *Server) ReceiveMessage(pb proto.Message) error {
 	case *internal.NodeEventMessage:
 		s.cluster.ReceiveEvent(DecodeNodeEvent(obj))
 	case *internal.NodeStatus:
-		s.HandleRemoteStatus(pb)
+		s.handleRemoteStatus(pb)
 	}
 
 	return nil
@@ -525,7 +524,7 @@ func (s *Server) SendSync(pb proto.Message) error {
 		node := node
 		s.logger.Printf("SendSync to: %s", node.URI)
 		// Don't forward the message to ourselves.
-		if s.URI == node.URI {
+		if s.uri == node.URI {
 			continue
 		}
 
@@ -548,44 +547,17 @@ func (s *Server) SendTo(to *Node, pb proto.Message) error {
 	return s.defaultClient.SendMessage(context.Background(), &to.URI, pb)
 }
 
-// Node returns the pilosa.Node object. It is used by membership protocols to
+// node returns the pilosa.node object. It is used by membership protocols to
 // get this node's name(ID), location(URI), and coordinator status.
-func (s *Server) Node() *Node {
-	return s.cluster.Node
+func (s *Server) node() Node {
+	return *s.cluster.Node
 }
 
-// Server implements StatusHandler.
-// LocalStatus is used to periodically sync information
-// between nodes. Under normal conditions, nodes should
-// remain in sync through Broadcast messages. For cases
-// where a node fails to receive a Broadcast message, or
-// when a new (empty) node needs to get in sync with the
-// rest of the cluster, two things are shared via gossip:
-// - MaxShard by Index
-// - Schema
-// In a gossip implementation, memberlist.Delegate.LocalState() uses this.
-func (s *Server) LocalStatus() (proto.Message, error) {
-	if s.cluster == nil {
-		return nil, errors.New("Server.Cluster is nil")
-	}
-	if s.holder == nil {
-		return nil, errors.New("Server.Holder is nil")
-	}
-
-	ns := internal.NodeStatus{
-		Node:      EncodeNode(s.cluster.Node),
-		MaxShards: s.holder.encodeMaxShards(),
-		Schema:    s.holder.encodeSchema(),
-	}
-
-	return &ns, nil
-}
-
-// HandleRemoteStatus receives incoming NodeStatus from remote nodes.
-func (s *Server) HandleRemoteStatus(pb proto.Message) error {
+// handleRemoteStatus receives incoming NodeStatus from remote nodes.
+func (s *Server) handleRemoteStatus(pb proto.Message) {
 	// Ignore NodeStatus messages until the cluster is in a Normal state.
 	if s.cluster.State() != ClusterStateNormal {
-		return nil
+		return
 	}
 
 	go func() {
@@ -597,8 +569,6 @@ func (s *Server) HandleRemoteStatus(pb proto.Message) error {
 			s.logger.Printf("merge remote status: %s", err)
 		}
 	}()
-
-	return nil
 }
 
 func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error {
@@ -643,7 +613,7 @@ func (s *Server) monitorDiagnostics() {
 
 	s.diagnostics.Logger = s.logger
 	s.diagnostics.SetVersion(Version)
-	s.diagnostics.Set("Host", s.URI.host)
+	s.diagnostics.Set("Host", s.uri.host)
 	s.diagnostics.Set("Cluster", strings.Join(s.cluster.nodeIDs(), ","))
 	s.diagnostics.Set("NumNodes", len(s.cluster.Nodes))
 	s.diagnostics.Set("NumCPU", runtime.NumCPU())
@@ -758,10 +728,3 @@ func expandDirName(path string) (string, error) {
 	}
 	return path, nil
 }
-
-type MemberServer interface {
-	ReceiveMessage(proto.Message) error
-	LocalStatus() (proto.Message, error)
-	HandleRemoteStatus(proto.Message) error
-	Node() *Node
-}
diff --git a/server/server.go b/server/server.go
index 7781ed4bc..c8de0664c 100644
--- a/server/server.go
+++ b/server/server.go
@@ -139,7 +139,7 @@ func (m *Command) Start() (err error) {
 		return errors.Wrap(err, "opening server")
 	}
 
-	m.logger.Printf("Listening as %s\n", m.Server.URI)
+	m.logger.Printf("Listening as %s\n", m.API.Node().URI)
 
 	return nil
 }
@@ -309,7 +309,7 @@ func (m *Command) SetupNetworking() error {
 	}
 
 	// get the host portion of addr to use for binding
-	gossipHost := m.Server.URI.Host()
+	gossipHost := m.API.Node().URI.Host()
 	m.gossipTransport, err = gossip.NewTransport(gossipHost, gossipPort, m.logger.Logger())
 	if err != nil {
 		return errors.Wrap(err, "getting transport")
diff --git a/server/server_test.go b/server/server_test.go
index 8b1e29acc..506807f70 100644
--- a/server/server_test.go
+++ b/server/server_test.go
@@ -44,7 +44,7 @@ func TestMain_Set_Quick(t *testing.T) {
 		defer m.Close()
 
 		// Create client.
-		client, err := http.NewInternalClient(m.Server.URI.HostPort(), http.GetHTTPClient(nil))
+		client, err := http.NewInternalClient(m.API.Node().URI.HostPort(), http.GetHTTPClient(nil))
 		if err != nil {
 			t.Fatal(err)
 		}
diff --git a/test/pilosa.go b/test/pilosa.go
index 4774b4cf8..3fa2a6b07 100644
--- a/test/pilosa.go
+++ b/test/pilosa.go
@@ -130,11 +130,11 @@ func (m *Command) Reopen() error {
 }
 
 // URL returns the base URL string for accessing the running program.
-func (m *Command) URL() string { return m.Server.URI.String() }
+func (m *Command) URL() string { return m.API.Node().URI.String() }
 
 // Client returns a client to connect to the program.
 func (m *Command) Client() *http.InternalClient {
-	client, err := http.NewInternalClient(m.Server.URI.HostPort(), http.GetHTTPClient(nil))
+	client, err := http.NewInternalClient(m.API.Node().URI.HostPort(), http.GetHTTPClient(nil))
 	if err != nil {
 		panic(err)
 	}

From f757e102552c56f009befcddaa5242884175cd0c Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Tue, 3 Jul 2018 16:28:51 -0500
Subject: [PATCH 04/32] support newlines in more places

---
 pql/pql.peg        |   7 +-
 pql/pql.peg.go     | 266 ++++++++++++++++++++-------------------------
 pql/pqlpeg_test.go |   7 ++
 3 files changed, 127 insertions(+), 153 deletions(-)

diff --git a/pql/pql.peg b/pql/pql.peg
index a33031543..141d8f265 100644
--- a/pql/pql.peg
+++ b/pql/pql.peg
@@ -5,7 +5,7 @@ type PQL Peg {
 }
 
 
-Calls <- whitesp (Call whitesp)* !.
+Calls <- sp (Call sp)* !.
 Call <-  'Set' {p.startCall("Set")} open col comma args (comma timestamp)? close {p.endCall()}
        / 'SetRowAttrs' {p.startCall("SetRowAttrs")} open posfield comma uintrow comma args close {p.endCall()}
        / 'SetColumnAttrs' {p.startCall("SetColumnAttrs")} open col comma args close {p.endCall()}
@@ -62,11 +62,10 @@ col <- ( <uint> {p.addPosNum("_col", buffer[begin:end])}
 
 open <- '(' sp
 close <- ')' sp
-sp <- ( ' ' / '\t' )*
-comma <- sp ',' whitesp
+sp <- ( ' ' / '\t' / '\n' )*
+comma <- sp ',' sp
 lbrack <- '[' sp
 rbrack <- sp ']' sp
-whitesp <- ( ' ' / '\t' / '\n' )*
 IDENT <- [[A-Z]] ([[A-Z]] / [0-9])*
 
 
diff --git a/pql/pql.peg.go b/pql/pql.peg.go
index 697d589c3..2516def6c 100644
--- a/pql/pql.peg.go
+++ b/pql/pql.peg.go
@@ -45,7 +45,6 @@ const (
 	rulecomma
 	rulelbrack
 	rulerbrack
-	rulewhitesp
 	ruleIDENT
 	ruletimestampbasicfmt
 	ruletimestampfmt
@@ -128,7 +127,6 @@ var rul3s = [...]string{
 	"comma",
 	"lbrack",
 	"rbrack",
-	"whitesp",
 	"IDENT",
 	"timestampbasicfmt",
 	"timestampfmt",
@@ -294,7 +292,7 @@ type PQL struct {
 
 	Buffer string
 	buffer []rune
-	rules  [80]func() bool
+	rules  [79]func() bool
 	parse  func(rule ...int) error
 	reset  func()
 	Pretty bool
@@ -548,12 +546,12 @@ func (p *PQL) Init() {
 
 	_rules = [...]func() bool{
 		nil,
-		/* 0 Calls <- <(whitesp (Call whitesp)* !.)> */
+		/* 0 Calls <- <(sp (Call sp)* !.)> */
 		func() bool {
 			position0, tokenIndex0 := position, tokenIndex
 			{
 				position1 := position
-				if !_rules[rulewhitesp]() {
+				if !_rules[rulesp]() {
 					goto l0
 				}
 			l2:
@@ -562,7 +560,7 @@ func (p *PQL) Init() {
 					if !_rules[ruleCall]() {
 						goto l3
 					}
-					if !_rules[rulewhitesp]() {
+					if !_rules[rulesp]() {
 						goto l3
 					}
 					goto l2
@@ -2525,7 +2523,7 @@ func (p *PQL) Init() {
 			position, tokenIndex = position257, tokenIndex257
 			return false
 		},
-		/* 25 sp <- <(' ' / '\t')*> */
+		/* 25 sp <- <(' ' / '\t' / '\n')*> */
 		func() bool {
 			{
 				position260 := position
@@ -2542,6 +2540,13 @@ func (p *PQL) Init() {
 					l264:
 						position, tokenIndex = position263, tokenIndex263
 						if buffer[position] != rune('\t') {
+							goto l265
+						}
+						position++
+						goto l263
+					l265:
+						position, tokenIndex = position263, tokenIndex263
+						if buffer[position] != rune('\n') {
 							goto l262
 						}
 						position++
@@ -2555,295 +2560,258 @@ func (p *PQL) Init() {
 			}
 			return true
 		},
-		/* 26 comma <- <(sp ',' whitesp)> */
+		/* 26 comma <- <(sp ',' sp)> */
 		func() bool {
-			position265, tokenIndex265 := position, tokenIndex
+			position266, tokenIndex266 := position, tokenIndex
 			{
-				position266 := position
+				position267 := position
 				if !_rules[rulesp]() {
-					goto l265
+					goto l266
 				}
 				if buffer[position] != rune(',') {
-					goto l265
+					goto l266
 				}
 				position++
-				if !_rules[rulewhitesp]() {
-					goto l265
+				if !_rules[rulesp]() {
+					goto l266
 				}
-				add(rulecomma, position266)
+				add(rulecomma, position267)
 			}
 			return true
-		l265:
-			position, tokenIndex = position265, tokenIndex265
+		l266:
+			position, tokenIndex = position266, tokenIndex266
 			return false
 		},
 		/* 27 lbrack <- <('[' sp)> */
 		nil,
 		/* 28 rbrack <- <(sp ']' sp)> */
 		nil,
-		/* 29 whitesp <- <(' ' / '\t' / '\n')*> */
-		func() bool {
-			{
-				position270 := position
-			l271:
-				{
-					position272, tokenIndex272 := position, tokenIndex
-					{
-						position273, tokenIndex273 := position, tokenIndex
-						if buffer[position] != rune(' ') {
-							goto l274
-						}
-						position++
-						goto l273
-					l274:
-						position, tokenIndex = position273, tokenIndex273
-						if buffer[position] != rune('\t') {
-							goto l275
-						}
-						position++
-						goto l273
-					l275:
-						position, tokenIndex = position273, tokenIndex273
-						if buffer[position] != rune('\n') {
-							goto l272
-						}
-						position++
-					}
-				l273:
-					goto l271
-				l272:
-					position, tokenIndex = position272, tokenIndex272
-				}
-				add(rulewhitesp, position270)
-			}
-			return true
-		},
-		/* 30 IDENT <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9])*)> */
+		/* 29 IDENT <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9])*)> */
 		nil,
-		/* 31 timestampbasicfmt <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> */
+		/* 30 timestampbasicfmt <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> */
 		func() bool {
-			position277, tokenIndex277 := position, tokenIndex
+			position271, tokenIndex271 := position, tokenIndex
 			{
-				position278 := position
+				position272 := position
 				if c := buffer[position]; c < rune('0') || c > rune('9') {
-					goto l277
+					goto l271
 				}
 				position++
 				if c := buffer[position]; c < rune('0') || c > rune('9') {
-					goto l277
+					goto l271
 				}
 				position++
 				if c := buffer[position]; c < rune('0') || c > rune('9') {
-					goto l277
+					goto l271
 				}
 				position++
 				if c := buffer[position]; c < rune('0') || c > rune('9') {
-					goto l277
+					goto l271
 				}
 				position++
 				if buffer[position] != rune('-') {
-					goto l277
+					goto l271
 				}
 				position++
 				{
-					position279, tokenIndex279 := position, tokenIndex
+					position273, tokenIndex273 := position, tokenIndex
 					if buffer[position] != rune('0') {
-						goto l280
+						goto l274
 					}
 					position++
-					goto l279
-				l280:
-					position, tokenIndex = position279, tokenIndex279
+					goto l273
+				l274:
+					position, tokenIndex = position273, tokenIndex273
 					if buffer[position] != rune('1') {
-						goto l277
+						goto l271
 					}
 					position++
 				}
-			l279:
+			l273:
 				if c := buffer[position]; c < rune('0') || c > rune('9') {
-					goto l277
+					goto l271
 				}
 				position++
 				if buffer[position] != rune('-') {
-					goto l277
+					goto l271
 				}
 				position++
 				if c := buffer[position]; c < rune('0') || c > rune('3') {
-					goto l277
+					goto l271
 				}
 				position++
 				if c := buffer[position]; c < rune('0') || c > rune('9') {
-					goto l277
+					goto l271
 				}
 				position++
 				if buffer[position] != rune('T') {
-					goto l277
+					goto l271
 				}
 				position++
 				if c := buffer[position]; c < rune('0') || c > rune('9') {
-					goto l277
+					goto l271
 				}
 				position++
 				if c := buffer[position]; c < rune('0') || c > rune('9') {
-					goto l277
+					goto l271
 				}
 				position++
 				if buffer[position] != rune(':') {
-					goto l277
+					goto l271
 				}
 				position++
 				if c := buffer[position]; c < rune('0') || c > rune('9') {
-					goto l277
+					goto l271
 				}
 				position++
 				if c := buffer[position]; c < rune('0') || c > rune('9') {
-					goto l277
+					goto l271
 				}
 				position++
-				add(ruletimestampbasicfmt, position278)
+				add(ruletimestampbasicfmt, position272)
 			}
 			return true
-		l277:
-			position, tokenIndex = position277, tokenIndex277
+		l271:
+			position, tokenIndex = position271, tokenIndex271
 			return false
 		},
-		/* 32 timestampfmt <- <(('"' timestampbasicfmt '"') / ('\'' timestampbasicfmt '\'') / timestampbasicfmt)> */
+		/* 31 timestampfmt <- <(('"' timestampbasicfmt '"') / ('\'' timestampbasicfmt '\'') / timestampbasicfmt)> */
 		func() bool {
-			position281, tokenIndex281 := position, tokenIndex
+			position275, tokenIndex275 := position, tokenIndex
 			{
-				position282 := position
+				position276 := position
 				{
-					position283, tokenIndex283 := position, tokenIndex
+					position277, tokenIndex277 := position, tokenIndex
 					if buffer[position] != rune('"') {
-						goto l284
+						goto l278
 					}
 					position++
 					if !_rules[ruletimestampbasicfmt]() {
-						goto l284
+						goto l278
 					}
 					if buffer[position] != rune('"') {
-						goto l284
+						goto l278
 					}
 					position++
-					goto l283
-				l284:
-					position, tokenIndex = position283, tokenIndex283
+					goto l277
+				l278:
+					position, tokenIndex = position277, tokenIndex277
 					if buffer[position] != rune('\'') {
-						goto l285
+						goto l279
 					}
 					position++
 					if !_rules[ruletimestampbasicfmt]() {
-						goto l285
+						goto l279
 					}
 					if buffer[position] != rune('\'') {
-						goto l285
+						goto l279
 					}
 					position++
-					goto l283
-				l285:
-					position, tokenIndex = position283, tokenIndex283
+					goto l277
+				l279:
+					position, tokenIndex = position277, tokenIndex277
 					if !_rules[ruletimestampbasicfmt]() {
-						goto l281
+						goto l275
 					}
 				}
-			l283:
-				add(ruletimestampfmt, position282)
+			l277:
+				add(ruletimestampfmt, position276)
 			}
 			return true
-		l281:
-			position, tokenIndex = position281, tokenIndex281
+		l275:
+			position, tokenIndex = position275, tokenIndex275
 			return false
 		},
-		/* 33 timestamp <- <(<timestampfmt> Action43)> */
+		/* 32 timestamp <- <(<timestampfmt> Action43)> */
 		nil,
-		/* 35 Action0 <- <{p.startCall("Set")}> */
+		/* 34 Action0 <- <{p.startCall("Set")}> */
 		nil,
-		/* 36 Action1 <- <{p.endCall()}> */
+		/* 35 Action1 <- <{p.endCall()}> */
 		nil,
-		/* 37 Action2 <- <{p.startCall("SetRowAttrs")}> */
+		/* 36 Action2 <- <{p.startCall("SetRowAttrs")}> */
 		nil,
-		/* 38 Action3 <- <{p.endCall()}> */
+		/* 37 Action3 <- <{p.endCall()}> */
 		nil,
-		/* 39 Action4 <- <{p.startCall("SetColumnAttrs")}> */
+		/* 38 Action4 <- <{p.startCall("SetColumnAttrs")}> */
 		nil,
-		/* 40 Action5 <- <{p.endCall()}> */
+		/* 39 Action5 <- <{p.endCall()}> */
 		nil,
-		/* 41 Action6 <- <{p.startCall("Clear")}> */
+		/* 40 Action6 <- <{p.startCall("Clear")}> */
 		nil,
-		/* 42 Action7 <- <{p.endCall()}> */
+		/* 41 Action7 <- <{p.endCall()}> */
 		nil,
-		/* 43 Action8 <- <{p.startCall("TopN")}> */
+		/* 42 Action8 <- <{p.startCall("TopN")}> */
 		nil,
-		/* 44 Action9 <- <{p.endCall()}> */
+		/* 43 Action9 <- <{p.endCall()}> */
 		nil,
-		/* 45 Action10 <- <{p.startCall("Range")}> */
+		/* 44 Action10 <- <{p.startCall("Range")}> */
 		nil,
-		/* 46 Action11 <- <{p.endCall()}> */
+		/* 45 Action11 <- <{p.endCall()}> */
 		nil,
 		nil,
-		/* 48 Action12 <- <{ p.startCall(buffer[begin:end] ) }> */
+		/* 47 Action12 <- <{ p.startCall(buffer[begin:end] ) }> */
 		nil,
-		/* 49 Action13 <- <{ p.endCall() }> */
+		/* 48 Action13 <- <{ p.endCall() }> */
 		nil,
-		/* 50 Action14 <- <{ p.addBTWN() }> */
+		/* 49 Action14 <- <{ p.addBTWN() }> */
 		nil,
-		/* 51 Action15 <- <{ p.addLTE() }> */
+		/* 50 Action15 <- <{ p.addLTE() }> */
 		nil,
-		/* 52 Action16 <- <{ p.addGTE() }> */
+		/* 51 Action16 <- <{ p.addGTE() }> */
 		nil,
-		/* 53 Action17 <- <{ p.addEQ() }> */
+		/* 52 Action17 <- <{ p.addEQ() }> */
 		nil,
-		/* 54 Action18 <- <{ p.addNEQ() }> */
+		/* 53 Action18 <- <{ p.addNEQ() }> */
 		nil,
-		/* 55 Action19 <- <{ p.addLT() }> */
+		/* 54 Action19 <- <{ p.addLT() }> */
 		nil,
-		/* 56 Action20 <- <{ p.addGT() }> */
+		/* 55 Action20 <- <{ p.addGT() }> */
 		nil,
-		/* 57 Action21 <- <{p.startConditional()}> */
+		/* 56 Action21 <- <{p.startConditional()}> */
 		nil,
-		/* 58 Action22 <- <{p.endConditional()}> */
+		/* 57 Action22 <- <{p.endConditional()}> */
 		nil,
-		/* 59 Action23 <- <{p.condAdd(buffer[begin:end])}> */
+		/* 58 Action23 <- <{p.condAdd(buffer[begin:end])}> */
 		nil,
-		/* 60 Action24 <- <{p.condAdd(buffer[begin:end])}> */
+		/* 59 Action24 <- <{p.condAdd(buffer[begin:end])}> */
 		nil,
-		/* 61 Action25 <- <{p.condAdd(buffer[begin:end])}> */
+		/* 60 Action25 <- <{p.condAdd(buffer[begin:end])}> */
 		nil,
-		/* 62 Action26 <- <{p.addPosStr("_start", buffer[begin:end])}> */
+		/* 61 Action26 <- <{p.addPosStr("_start", buffer[begin:end])}> */
 		nil,
-		/* 63 Action27 <- <{p.addPosStr("_end", buffer[begin:end])}> */
+		/* 62 Action27 <- <{p.addPosStr("_end", buffer[begin:end])}> */
 		nil,
-		/* 64 Action28 <- <{ p.startList() }> */
+		/* 63 Action28 <- <{ p.startList() }> */
 		nil,
-		/* 65 Action29 <- <{ p.endList() }> */
+		/* 64 Action29 <- <{ p.endList() }> */
 		nil,
-		/* 66 Action30 <- <{ p.addVal(nil) }> */
+		/* 65 Action30 <- <{ p.addVal(nil) }> */
 		nil,
-		/* 67 Action31 <- <{ p.addVal(true) }> */
+		/* 66 Action31 <- <{ p.addVal(true) }> */
 		nil,
-		/* 68 Action32 <- <{ p.addVal(false) }> */
+		/* 67 Action32 <- <{ p.addVal(false) }> */
 		nil,
-		/* 69 Action33 <- <{ p.addNumVal(buffer[begin:end]) }> */
+		/* 68 Action33 <- <{ p.addNumVal(buffer[begin:end]) }> */
 		nil,
-		/* 70 Action34 <- <{ p.addNumVal(buffer[begin:end]) }> */
+		/* 69 Action34 <- <{ p.addNumVal(buffer[begin:end]) }> */
 		nil,
-		/* 71 Action35 <- <{ p.addVal(buffer[begin:end]) }> */
+		/* 70 Action35 <- <{ p.addVal(buffer[begin:end]) }> */
 		nil,
-		/* 72 Action36 <- <{ p.addVal(buffer[begin:end]) }> */
+		/* 71 Action36 <- <{ p.addVal(buffer[begin:end]) }> */
 		nil,
-		/* 73 Action37 <- <{ p.addVal(buffer[begin:end]) }> */
+		/* 72 Action37 <- <{ p.addVal(buffer[begin:end]) }> */
 		nil,
-		/* 74 Action38 <- <{ p.addField(buffer[begin:end]) }> */
+		/* 73 Action38 <- <{ p.addField(buffer[begin:end]) }> */
 		nil,
-		/* 75 Action39 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */
+		/* 74 Action39 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */
 		nil,
-		/* 76 Action40 <- <{p.addPosNum("_row", buffer[begin:end])}> */
+		/* 75 Action40 <- <{p.addPosNum("_row", buffer[begin:end])}> */
 		nil,
-		/* 77 Action41 <- <{p.addPosNum("_col", buffer[begin:end])}> */
+		/* 76 Action41 <- <{p.addPosNum("_col", buffer[begin:end])}> */
 		nil,
-		/* 78 Action42 <- <{p.addPosStr("_col", buffer[begin:end])}> */
+		/* 77 Action42 <- <{p.addPosStr("_col", buffer[begin:end])}> */
 		nil,
-		/* 79 Action43 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */
+		/* 78 Action43 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */
 		nil,
 	}
 	p.rules = _rules
diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go
index f472e2cc2..ad40364b3 100644
--- a/pql/pqlpeg_test.go
+++ b/pql/pqlpeg_test.go
@@ -228,6 +228,13 @@ func TestPEGWorking(t *testing.T) {
 			name:   "Dashed Frame",
 			input:  "Set(1, my-frame=9)",
 			ncalls: 1},
+		{
+			name: "newlines",
+			input: `Set(
+1,
+my-frame
+=9)`,
+			ncalls: 1},
 	}
 
 	for i, test := range tests {

From 4ef8fff9b6dbce49f951bf4c3e75481c3b2b7327 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Wed, 4 Jul 2018 07:29:16 -0500
Subject: [PATCH 05/32] WIP, broken. refactoring to isolate intneral structs
 and define core structs

---
 broadcast.go |  33 ++++--
 cluster.go   | 306 ++++++++++++++++++++++++++++++++++++++-------------
 server.go    |   6 +-
 3 files changed, 259 insertions(+), 86 deletions(-)

diff --git a/broadcast.go b/broadcast.go
index 19e927c18..3f6b960e4 100644
--- a/broadcast.go
+++ b/broadcast.go
@@ -25,11 +25,15 @@ import (
 
 // broadcaster is an interface for broadcasting messages.
 type broadcaster interface {
-	SendSync(pb proto.Message) error
-	SendAsync(pb proto.Message) error
-	SendTo(to *Node, pb proto.Message) error
+	SendSync(Message) error
+	SendAsync(Message) error
+	SendTo(*Node, Message) error
 }
 
+// Message is the interface implemented by all core pilosa types which can be serialized to messages.
+// TODO add at least a single "isMessage()" method.
+type Message interface{}
+
 func init() {
 	NopBroadcaster = &nopBroadcaster{}
 }
@@ -40,13 +44,13 @@ var NopBroadcaster broadcaster
 type nopBroadcaster struct{}
 
 // SendSync A no-op implementation of Broadcaster SendSync method.
-func (n nopBroadcaster) SendSync(pb proto.Message) error { return nil }
+func (nopBroadcaster) SendSync(Message) error { return nil }
 
 // SendAsync A no-op implementation of Broadcaster SendAsync method.
-func (n nopBroadcaster) SendAsync(pb proto.Message) error { return nil }
+func (nopBroadcaster) SendAsync(Message) error { return nil }
 
 // SendTo is a no-op implementation of Broadcaster SendTo method.
-func (c nopBroadcaster) SendTo(to *Node, pb proto.Message) error { return nil }
+func (nopBroadcaster) SendTo(*Node, Message) error { return nil }
 
 // Broadcast message types.
 const (
@@ -69,7 +73,8 @@ const (
 )
 
 // MarshalMessage encodes the protobuf message into a byte slice.
-func MarshalMessage(m proto.Message) ([]byte, error) {
+func MarshalMessage(pm Message) ([]byte, error) {
+	m := encode(pm)
 	var typ uint8
 	switch obj := m.(type) {
 	case *internal.CreateShardMessage:
@@ -114,8 +119,20 @@ func MarshalMessage(m proto.Message) ([]byte, error) {
 	return append([]byte{typ}, buf...), nil
 }
 
+func encode(m Message) proto.Message {
+	var pm proto.Message
+	switch mt := m.(type) {
+	case *CreateShardMessage:
+		return encodeCreateShardMessage(mt)
+	case *CreateIndexMessage:
+		return encodeCreateIndexMessage(mt)
+		// TODO, the rest
+	}
+	return nil
+}
+
 // UnmarshalMessage decodes the byte slice into a protobuf message.
-func UnmarshalMessage(buf []byte) (proto.Message, error) {
+func UnmarshalMessage(buf []byte) (Message, error) {
 	typ, buf := buf[0], buf[1:]
 	var m proto.Message
 	switch typ {
diff --git a/cluster.go b/cluster.go
index acf06dae0..fcfb08d73 100644
--- a/cluster.go
+++ b/cluster.go
@@ -69,52 +69,6 @@ func (n Node) String() string {
 	return fmt.Sprintf("Node: %s", n.ID)
 }
 
-// EncodeNodes converts a slice of Nodes into its internal representation.
-func EncodeNodes(a []*Node) []*internal.Node {
-	other := make([]*internal.Node, len(a))
-	for i := range a {
-		other[i] = EncodeNode(a[i])
-	}
-	return other
-}
-
-// EncodeNode converts a Node into its internal representation.
-func EncodeNode(n *Node) *internal.Node {
-	return &internal.Node{
-		ID:            n.ID,
-		URI:           n.URI.Encode(),
-		IsCoordinator: n.IsCoordinator,
-	}
-}
-
-// DecodeNodes converts a proto message into a slice of Nodes.
-func DecodeNodes(a []*internal.Node) []*Node {
-	if len(a) == 0 {
-		return nil
-	}
-	other := make([]*Node, len(a))
-	for i := range a {
-		other[i] = DecodeNode(a[i])
-	}
-	return other
-}
-
-// DecodeNode converts a proto message into a Node.
-func DecodeNode(node *internal.Node) *Node {
-	return &Node{
-		ID:            node.ID,
-		URI:           decodeURI(node.URI),
-		IsCoordinator: node.IsCoordinator,
-	}
-}
-
-func DecodeNodeEvent(ne *internal.NodeEventMessage) *nodeEvent {
-	return &nodeEvent{
-		Event: NodeEventType(ne.Event),
-		Node:  DecodeNode(ne.Node),
-	}
-}
-
 // Nodes represents a list of nodes.
 type Nodes []*Node
 
@@ -1176,7 +1130,7 @@ func (c *cluster) completeCurrentJob(state string) error {
 }
 
 // followResizeInstruction is run by any node that receives a ResizeInstruction.
-func (c *cluster) followResizeInstruction(instr *internal.ResizeInstruction) error {
+func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error {
 	c.logger.Printf("follow resize instruction on %s", c.Node.ID)
 	// Make sure the cluster status on this node agrees with the Coordinator
 	// before attempting a resize.
@@ -1553,32 +1507,6 @@ func (c *cluster) saveTopology() error {
 	return nil
 }
 
-func encodeTopology(topology *Topology) *internal.Topology {
-	if topology == nil {
-		return nil
-	}
-	return &internal.Topology{
-		ClusterID: topology.ClusterID,
-		NodeIDs:   topology.NodeIDs,
-	}
-}
-
-func decodeTopology(topology *internal.Topology) (*Topology, error) {
-	if topology == nil {
-		return nil, nil
-	}
-
-	t := NewTopology()
-	t.ClusterID = topology.ClusterID
-	t.NodeIDs = topology.NodeIDs
-	sort.Slice(t.NodeIDs,
-		func(i, j int) bool {
-			return t.NodeIDs[i] < t.NodeIDs[j]
-		})
-
-	return t, nil
-}
-
 func (c *cluster) considerTopology() error {
 	// Create ClusterID if one does not already exist.
 	if c.id == "" {
@@ -1752,7 +1680,7 @@ func (c *cluster) nodeLeave(node *Node) error {
 	return nil
 }
 
-func (c *cluster) mergeClusterStatus(cs *internal.ClusterStatus) error {
+func (c *cluster) mergeClusterStatus(cs *ClusterStatus) error {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	c.logger.Printf("merge cluster status: %v", cs)
@@ -1764,7 +1692,7 @@ func (c *cluster) mergeClusterStatus(cs *internal.ClusterStatus) error {
 	// Set ClusterID.
 	c.setID(cs.ClusterID)
 
-	officialNodes := DecodeNodes(cs.Nodes)
+	officialNodes := cs.Nodes
 
 	// Add all nodes from the coordinator.
 	for _, node := range officialNodes {
@@ -1813,3 +1741,231 @@ func (c *cluster) setStatic(hosts []string) error {
 	}
 	return nil
 }
+
+type ClusterStatus struct {
+	ClusterID string
+	State     string
+	Nodes     []*Node
+}
+
+type ResizeInstruction struct {
+	JobID         int64
+	Node          *Node
+	Coordinator   *Node
+	Sources       []*ResizeSource
+	Schema        *Schema
+	ClusterStatus *ClusterStatus
+}
+
+func decodeResizeInstruction(ri *internal.ResizeInstruction) ResizeInstruction {
+	return &ResizeInstruction{
+		JobID:         ri.JobID,
+		Node:          DecodeNode(ri.Node),
+		Coordinator:   DecodeNode(ri.Coordinator),
+		Sources:       decodeResizeSources(ri.Sources),
+		Schema:        decodeSchema(ri.Schema),
+		ClusterStatus: decodeClusterStatus(ri.ClusterStatus),
+	}
+}
+
+type ResizeSource struct {
+	Node  *Node  `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"`
+	Index string `protobuf:"bytes,2,opt,name=Index,proto3" json:"Index,omitempty"`
+	Field string `protobuf:"bytes,3,opt,name=Field,proto3" json:"Field,omitempty"`
+	View  string `protobuf:"bytes,4,opt,name=View,proto3" json:"View,omitempty"`
+	Shard uint64 `protobuf:"varint,5,opt,name=Shard,proto3" json:"Shard,omitempty"`
+}
+
+func decodeResizeSources(srcs []*internal.ResizeSource) []*ResizeSource {
+	new := make([]*ResizeSource, 0, len(srcs))
+	for _, src := range srcs {
+		new = append(new, decodeResizeSource(src))
+	}
+	return new
+}
+
+func decodeResizeSource(rs *internal.ResizeSource) *ResizeSource {
+	return &ResizeSource{
+		Node:  DecodeNode(rs.Node),
+		Index: rs.Index,
+		Field: rs.Field,
+		View:  rs.View,
+		Shard: rs.Shard,
+	}
+}
+
+// Schema is a schema
+type Schema struct {
+	Indexes []*IndexInfo
+}
+
+func decodeSchema(s *internal.Schema) *Schema {
+	return &Schema{
+		Indexes: decodeIndexes(s.Indexes),
+	}
+}
+
+func decodeIndexes(idxs []*internal.Index) []*IndexInfo {
+	new := make([]*IndexInfo, 0, len(idxs))
+	for _, idx := range idxs {
+		new = append(new, decodeIndex(idx))
+	}
+	return new
+}
+
+func decodeIndex(idx *internal.Index) *IndexInfo {
+	return &IndexInfo{
+		Name:   idx.Name,
+		Fields: decodeFields(idx.Fields),
+	}
+}
+
+func decodeFields(fs []*internal.Field) []*FieldInfo {
+	new := make([]*FieldInfo, 0, len(fs))
+	for _, f := range fs {
+		new = append(new, decodeField(f))
+	}
+	return new
+}
+
+func decodeField(f *internal.Field) *FieldInfo {
+	fi := &FieldInfo{
+		Name:    f.Name,
+		Options: *decodeFieldOptions(f.Meta),
+		Views:   make([]*viewInfo, 0, len(f.Views)),
+	}
+	for _, viewname := range f.Views {
+		fi.Views = append(fi.Views, &viewInfo{Name: viewname})
+	}
+	return fi
+}
+
+// EncodeNodes converts a slice of Nodes into its internal representation.
+func EncodeNodes(a []*Node) []*internal.Node {
+	other := make([]*internal.Node, len(a))
+	for i := range a {
+		other[i] = EncodeNode(a[i])
+	}
+	return other
+}
+
+// EncodeNode converts a Node into its internal representation.
+func EncodeNode(n *Node) *internal.Node {
+	return &internal.Node{
+		ID:            n.ID,
+		URI:           n.URI.Encode(),
+		IsCoordinator: n.IsCoordinator,
+	}
+}
+
+// DecodeNodes converts a proto message into a slice of Nodes.
+func DecodeNodes(a []*internal.Node) []*Node {
+	if len(a) == 0 {
+		return nil
+	}
+	other := make([]*Node, len(a))
+	for i := range a {
+		other[i] = DecodeNode(a[i])
+	}
+	return other
+}
+
+func decodeClusterStatus(cs *internal.ClusterStatus) *ClusterStatus {
+	return &ClusterStatus{
+		State:     cs.State,
+		ClusterID: cs.ClusterID,
+		Nodes:     DecodeNodes(cs.Nodes),
+	}
+}
+
+// DecodeNode converts a proto message into a Node.
+func DecodeNode(node *internal.Node) *Node {
+	return &Node{
+		ID:            node.ID,
+		URI:           decodeURI(node.URI),
+		IsCoordinator: node.IsCoordinator,
+	}
+}
+
+func DecodeNodeEvent(ne *internal.NodeEventMessage) *nodeEvent {
+	return &nodeEvent{
+		Event: NodeEventType(ne.Event),
+		Node:  DecodeNode(ne.Node),
+	}
+}
+
+func encodeTopology(topology *Topology) *internal.Topology {
+	if topology == nil {
+		return nil
+	}
+	return &internal.Topology{
+		ClusterID: topology.ClusterID,
+		NodeIDs:   topology.NodeIDs,
+	}
+}
+
+func decodeTopology(topology *internal.Topology) (*Topology, error) {
+	if topology == nil {
+		return nil, nil
+	}
+
+	t := NewTopology()
+	t.ClusterID = topology.ClusterID
+	t.NodeIDs = topology.NodeIDs
+	sort.Slice(t.NodeIDs,
+		func(i, j int) bool {
+			return t.NodeIDs[i] < t.NodeIDs[j]
+		})
+
+	return t, nil
+}
+
+type CreateShardMessage struct {
+	Index string
+	Shard uint64
+}
+
+func encodeCreateShardMessage(m *CreateShardMessage) *internal.CreateShardMessage {
+	return &internal.CreateShardMessage{
+		Index: m.Index,
+		Shard: m.Shard,
+	}
+}
+
+func decodeCreateShardMessage(pb *internal.CreateShardMessage) *CreateShardMessage {
+	return &CreateShardMessage{
+		Index: pb.Index,
+		Shard: pb.Shard,
+	}
+}
+
+type CreateIndexMessage struct {
+	Index string
+	Meta  *IndexOptions
+}
+
+func encodeCreateIndexMessage(m *CreateIndexMessage) *internal.CreateIndexMessage {
+	return &internal.CreateIndexMessage{
+		Index: m.Index,
+		Meta:  encodeIndexMeta(m.Meta),
+	}
+}
+
+func decodeCreateIndexMessage(pb *internal.CreateIndexMessage) *CreateIndexMessage {
+	return &CreateIndexMessage{
+		Index: pb.Index,
+		Meta:  decodeIndexMeta(pb.Meta),
+	}
+}
+
+func encodeIndexMeta(m *IndexOptions) *internal.IndexMeta {
+	return &internal.IndexMeta{
+		Keys: m.Keys,
+	}
+}
+
+func decodeIndexMeta(pb *internal.IndexMeta) *IndexOptions {
+	return &IndexOptions{
+		Keys: pb.Keys,
+	}
+}
diff --git a/server.go b/server.go
index 56195b427..dffeb94a0 100644
--- a/server.go
+++ b/server.go
@@ -483,12 +483,12 @@ func (s *Server) receiveMessage(pb proto.Message) error {
 			return err
 		}
 	case *internal.ClusterStatus:
-		err := s.cluster.mergeClusterStatus(obj)
+		err := s.cluster.mergeClusterStatus(decodeClusterStatus(obj))
 		if err != nil {
 			return err
 		}
 	case *internal.ResizeInstruction:
-		err := s.cluster.followResizeInstruction(obj)
+		err := s.cluster.followResizeInstruction(decodeResizeInstruction(obj))
 		if err != nil {
 			return err
 		}
@@ -518,7 +518,7 @@ func (s *Server) receiveMessage(pb proto.Message) error {
 }
 
 // SendSync represents an implementation of Broadcaster.
-func (s *Server) SendSync(pb proto.Message) error {
+func (s *Server) SendSync(m Message) error {
 	var eg errgroup.Group
 	for _, node := range s.cluster.Nodes {
 		node := node

From f526f18d82f8026d348d0966960fda6661c69232 Mon Sep 17 00:00:00 2001
From: Travis Turner <travis@pilosa.com>
Date: Wed, 4 Jul 2018 10:59:08 -0500
Subject: [PATCH 06/32] rename internal_tests. fix license header

---
 iterator_test.go => iterator_internal_test.go |  0
 uri_test.go => uri_internal_test.go           | 36 +++++--------------
 2 files changed, 9 insertions(+), 27 deletions(-)
 rename iterator_test.go => iterator_internal_test.go (100%)
 rename uri_test.go => uri_internal_test.go (74%)

diff --git a/iterator_test.go b/iterator_internal_test.go
similarity index 100%
rename from iterator_test.go
rename to iterator_internal_test.go
diff --git a/uri_test.go b/uri_internal_test.go
similarity index 74%
rename from uri_test.go
rename to uri_internal_test.go
index 70ea3f273..dbcbfa04d 100644
--- a/uri_test.go
+++ b/uri_internal_test.go
@@ -1,34 +1,16 @@
 // Copyright 2017 Pilosa Corp.
 //
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions
-// are met:
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
 //
-// 1. Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
+//     http://www.apache.org/licenses/LICENSE-2.0
 //
-// 2. Redistributions in binary form must reproduce the above copyright
-// notice, this list of conditions and the following disclaimer in the
-// documentation and/or other materials provided with the distribution.
-//
-// 3. Neither the name of the copyright holder nor the names of its
-// contributors may be used to endorse or promote products derived
-// from this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
-// CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
-// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
-// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
-// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
-// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
-// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
-// DAMAGE.
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
 
 package pilosa
 

From 1fb8330c8cf0d351c228a1345db19fa0506ee186 Mon Sep 17 00:00:00 2001
From: Travis Turner <travis@pilosa.com>
Date: Wed, 4 Jul 2018 12:27:42 -0500
Subject: [PATCH 07/32] remove some dead code highlighed by the unexport script

---
 api.go                | 57 ++-----------------------------------------
 apimethod_string.go   |  4 +--
 cluster.go            |  3 +--
 field.go              |  7 ------
 row.go                | 28 ---------------------
 time.go               |  9 -------
 time_internal_test.go | 16 +++++++++---
 uri.go                | 12 ++-------
 uri_internal_test.go  | 10 --------
 9 files changed, 20 insertions(+), 126 deletions(-)

diff --git a/api.go b/api.go
index abf2a84ea..be75624bc 100644
--- a/api.go
+++ b/api.go
@@ -370,55 +370,6 @@ func (api *API) ShardNodes(ctx context.Context, indexName string, shard uint64)
 	return api.cluster.shardNodes(indexName, shard), nil
 }
 
-// MarshalFragment returns an object which can write the specified fragment's data
-// to an io.Writer. The serialized data can be read back into a fragment with
-// the UnmarshalFragment API call.
-func (api *API) MarshalFragment(ctx context.Context, indexName string, fieldName string, shard uint64) (io.WriterTo, error) {
-	if err := api.validate(apiMarshalFragment); err != nil {
-		return nil, errors.Wrap(err, "validating api method")
-	}
-
-	// Retrieve fragment from holder.
-	f := api.holder.fragment(indexName, fieldName, viewStandard, shard)
-	if f == nil {
-		return nil, ErrFragmentNotFound
-	}
-	return f, nil
-}
-
-// UnmarshalFragment creates a new fragment (if necessary) and reads data from a
-// Reader which was previously written by MarshalFragment to populate the
-// fragment's data.
-func (api *API) UnmarshalFragment(ctx context.Context, indexName string, fieldName string, shard uint64, reader io.ReadCloser) error {
-	if err := api.validate(apiUnmarshalFragment); err != nil {
-		return errors.Wrap(err, "validating api method")
-	}
-
-	// Retrieve field.
-	f := api.holder.Field(indexName, fieldName)
-	if f == nil {
-		return ErrFieldNotFound
-	}
-
-	// Retrieve view.
-	view, err := f.createViewIfNotExists(viewStandard)
-	if err != nil {
-		return errors.Wrap(err, "creating view")
-	}
-
-	// Retrieve fragment from field.
-	frag, err := view.CreateFragmentIfNotExists(shard)
-	if err != nil {
-		return errors.Wrap(err, "creating fragment")
-	}
-
-	// Read fragment in from request body.
-	if _, err := frag.ReadFrom(reader); err != nil {
-		return errors.Wrap(err, "reading fragment")
-	}
-	return nil
-}
-
 // FragmentBlockData is an endpoint for internal usage. It is not guaranteed to
 // return anything useful. Currently it returns protobuf encoded row and column
 // ids from a "block" which is a subdivision of a fragment.
@@ -889,7 +840,6 @@ const (
 	apiIndexAttrDiff
 	//apiLocalID // not implemented
 	//apiLongQueryTime // not implemented
-	apiMarshalFragment
 	//apiMaxShards // not implemented
 	apiQuery
 	apiRecalculateCaches
@@ -900,15 +850,13 @@ const (
 	apiShardNodes
 	//apiState // not implemented
 	//apiStatsWithTags // not implemented
-	apiUnmarshalFragment
 	//apiVersion // not implemented
 	apiViews
 )
 
 var methodsCommon = map[apiMethod]struct{}{
-	apiClusterMessage:  struct{}{},
-	apiMarshalFragment: struct{}{},
-	apiSetCoordinator:  struct{}{},
+	apiClusterMessage: struct{}{},
+	apiSetCoordinator: struct{}{},
 }
 
 var methodsResizing = map[apiMethod]struct{}{
@@ -934,6 +882,5 @@ var methodsNormal = map[apiMethod]struct{}{
 	apiRecalculateCaches: struct{}{},
 	apiRemoveNode:        struct{}{},
 	apiShardNodes:        struct{}{},
-	apiUnmarshalFragment: struct{}{},
 	apiViews:             struct{}{},
 }
diff --git a/apimethod_string.go b/apimethod_string.go
index 881b79472..01217092f 100644
--- a/apimethod_string.go
+++ b/apimethod_string.go
@@ -4,9 +4,9 @@ package pilosa
 
 import "strconv"
 
-const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFieldapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiMarshalFragmentapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiShardNodesapiUnmarshalFragmentapiViews"
+const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFieldapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiShardNodesapiViews"
 
-var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 73, 86, 98, 118, 135, 143, 159, 168, 182, 190, 206, 224, 232, 252, 265, 279, 296, 309, 329, 337}
+var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 73, 86, 98, 118, 135, 143, 159, 168, 182, 190, 206, 214, 234, 247, 261, 278, 291, 299}
 
 func (i apiMethod) String() string {
 	if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) {
diff --git a/cluster.go b/cluster.go
index acf06dae0..3fe0dbfa5 100644
--- a/cluster.go
+++ b/cluster.go
@@ -45,8 +45,7 @@ const (
 	ClusterStateResizing = "RESIZING"
 
 	// NodeState represents the state of a node during startup.
-	NodeStateLoading = "LOADING"
-	NodeStateReady   = "READY"
+	NodeStateReady = "READY"
 
 	// resizeJob states.
 	resizeJobStateRunning = "RUNNING"
diff --git a/field.go b/field.go
index 76be9cf3f..56faffcf1 100644
--- a/field.go
+++ b/field.go
@@ -178,13 +178,6 @@ func (f *Field) Type() string {
 	return f.options.Type
 }
 
-// CacheType returns the caching mode for the field.
-func (f *Field) CacheType() string {
-	f.mu.RLock()
-	defer f.mu.RUnlock()
-	return f.options.CacheType
-}
-
 // SetCacheSize sets the cache size for ranked fames. Persists to meta file on update.
 // defaults to DefaultCacheSize 50000
 func (f *Field) SetCacheSize(v uint32) error {
diff --git a/row.go b/row.go
index cbfa6b270..4c722c936 100644
--- a/row.go
+++ b/row.go
@@ -216,25 +216,6 @@ func (r *Row) InvalidateCount() {
 	}
 }
 
-// IncrementCount increments the row cached counter, note this is an optimization that assumes that the caller is aware the size increased.
-func (r *Row) IncrementCount(i uint64) {
-	seg := r.segment(i / ShardWidth)
-	if seg != nil {
-		seg.n++
-	}
-
-}
-
-// DecrementCount decrements the row cached counter.
-func (r *Row) DecrementCount(i uint64) {
-	seg := r.segment(i / ShardWidth)
-	if seg != nil {
-		if seg.n > 0 {
-			seg.n--
-		}
-	}
-}
-
 // Count returns the number of columns in the row.
 func (r *Row) Count() uint64 {
 	var n uint64
@@ -297,15 +278,6 @@ func DecodeRow(pr *internal.Row) *Row {
 	return r
 }
 
-// Union performs a union on a slice of rows.
-func Union(rows []*Row) *Row {
-	other := rows[0]
-	for _, r := range rows[1:] {
-		other = other.Union(r)
-	}
-	return other
-}
-
 // RowSegment holds a subset of a row.
 // This could point to a mmapped roaring bitmap or an in-memory bitmap. The
 // width of the segment will always match the shard width.
diff --git a/time.go b/time.go
index def889304..ecfdcad3d 100644
--- a/time.go
+++ b/time.go
@@ -70,15 +70,6 @@ func (q TimeQuantum) Type() string {
 	return "TimeQuantum"
 }
 
-// ParseTimeQuantum parses v into a time quantum.
-func ParseTimeQuantum(v string) (TimeQuantum, error) {
-	q := TimeQuantum(strings.ToUpper(v))
-	if !q.Valid() {
-		return "", ErrInvalidTimeQuantum
-	}
-	return q, nil
-}
-
 // viewByTimeUnit returns the view name for time with a given quantum unit.
 func viewByTimeUnit(name string, t time.Time, unit rune) string {
 	switch unit {
diff --git a/time_internal_test.go b/time_internal_test.go
index 2920685fc..bd4afae43 100644
--- a/time_internal_test.go
+++ b/time_internal_test.go
@@ -16,6 +16,7 @@ package pilosa
 
 import (
 	"reflect"
+	"strings"
 	"testing"
 	"time"
 )
@@ -23,7 +24,7 @@ import (
 // Ensure string can be parsed into time quantum.
 func TestParseTimeQuantum(t *testing.T) {
 	t.Run("OK", func(t *testing.T) {
-		if q, err := ParseTimeQuantum("YMDH"); err != nil {
+		if q, err := parseTimeQuantum("YMDH"); err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		} else if q != TimeQuantum("YMDH") {
 			t.Fatalf("unexpected quantum: %#v", q)
@@ -31,7 +32,7 @@ func TestParseTimeQuantum(t *testing.T) {
 	})
 
 	t.Run("ErrInvalidTimeQuantum", func(t *testing.T) {
-		if _, err := ParseTimeQuantum("BADQUANTUM"); err != ErrInvalidTimeQuantum {
+		if _, err := parseTimeQuantum("BADQUANTUM"); err != ErrInvalidTimeQuantum {
 			t.Fatalf("unexpected error: %s", err)
 		}
 	})
@@ -160,9 +161,18 @@ func mustParseTime(value string) time.Time {
 
 // mustParseTimeQuantum parses v into a time quantum. Panic on error.
 func mustParseTimeQuantum(v string) TimeQuantum {
-	q, err := ParseTimeQuantum(v)
+	q, err := parseTimeQuantum(v)
 	if err != nil {
 		panic(err)
 	}
 	return q
 }
+
+// parseTimeQuantum parses v into a time quantum.
+func parseTimeQuantum(v string) (TimeQuantum, error) {
+	q := TimeQuantum(strings.ToUpper(v))
+	if !q.Valid() {
+		return "", ErrInvalidTimeQuantum
+	}
+	return q, nil
+}
diff --git a/uri.go b/uri.go
index 5d823d8b6..4eac6253e 100644
--- a/uri.go
+++ b/uri.go
@@ -148,14 +148,6 @@ func (u URI) String() string {
 	return fmt.Sprintf("%s://%s:%d", u.scheme, u.host, u.port)
 }
 
-// Equals returns true if the checked URI is equivalent to this URI.
-func (u URI) Equals(other *URI) bool {
-	if other == nil {
-		return false
-	}
-	return u == *other
-}
-
 // Path returns URI with path
 func (u *URI) Path(path string) string {
 	return fmt.Sprintf("%s%s", u.Normalize(), path)
@@ -163,7 +155,7 @@ func (u *URI) Path(path string) string {
 
 // The following methods are required to implement pflag Value interface.
 
-// Set sets the time quantum value.
+// Set sets the uri value.
 func (u *URI) Set(value string) error {
 	uri, err := NewURIFromAddress(value)
 	if err != nil {
@@ -173,7 +165,7 @@ func (u *URI) Set(value string) error {
 	return nil
 }
 
-// Type returns the type of a time quantum value.
+// Type returns the type of a uri.
 func (u URI) Type() string {
 	return "URI"
 }
diff --git a/uri_internal_test.go b/uri_internal_test.go
index dbcbfa04d..37dbddb70 100644
--- a/uri_internal_test.go
+++ b/uri_internal_test.go
@@ -76,16 +76,6 @@ func TestURIPath(t *testing.T) {
 	}
 }
 
-func TestEquals(t *testing.T) {
-	uri1 := DefaultURI()
-	if uri1.Equals(nil) {
-		t.Fatalf("URI should not be equal to nil")
-	}
-	if !uri1.Equals(DefaultURI()) {
-		t.Fatalf("URI should be equal to another URI with the same scheme, host and port")
-	}
-}
-
 func TestSetScheme(t *testing.T) {
 	uri := DefaultURI()
 	target := "fun"

From 6417f468bb70a292d97bdcf6d7897d353440d5e0 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Wed, 4 Jul 2018 16:50:20 -0500
Subject: [PATCH 08/32] wip implement more... still quite broken

---
 api.go                 |   2 +-
 broadcast.go           |  73 ++++++++-
 cluster.go             | 357 +++++++++++++++++++++++++++++++++++++----
 holder.go              |   7 +-
 server.go              |  68 ++++----
 utils_internal_test.go |  27 ++--
 6 files changed, 447 insertions(+), 87 deletions(-)

diff --git a/api.go b/api.go
index abf2a84ea..65ff90870 100644
--- a/api.go
+++ b/api.go
@@ -517,7 +517,7 @@ func (api *API) ClusterMessage(ctx context.Context, reqBody io.Reader) error {
 	}
 
 	// Forward the error message.
-	if err := api.server.receiveMessage(pb); err != nil {
+	if err := api.server.receiveMessage(decode(pb)); err != nil {
 		return errors.Wrap(err, "receiving message")
 	}
 	return nil
diff --git a/broadcast.go b/broadcast.go
index 3f6b960e4..0a33d4ca7 100644
--- a/broadcast.go
+++ b/broadcast.go
@@ -73,8 +73,7 @@ const (
 )
 
 // MarshalMessage encodes the protobuf message into a byte slice.
-func MarshalMessage(pm Message) ([]byte, error) {
-	m := encode(pm)
+func MarshalMessage(m proto.Message) ([]byte, error) {
 	var typ uint8
 	switch obj := m.(type) {
 	case *internal.CreateShardMessage:
@@ -120,19 +119,45 @@ func MarshalMessage(pm Message) ([]byte, error) {
 }
 
 func encode(m Message) proto.Message {
-	var pm proto.Message
 	switch mt := m.(type) {
 	case *CreateShardMessage:
 		return encodeCreateShardMessage(mt)
 	case *CreateIndexMessage:
 		return encodeCreateIndexMessage(mt)
-		// TODO, the rest
+	case *DeleteIndexMessage:
+		return encodeDeleteIndexMessage(mt)
+	case *CreateFieldMessage:
+		return encodeCreateFieldMessage(mt)
+	case *DeleteFieldMessage:
+		return encodeDeleteFieldMessage(mt)
+	case *CreateViewMessage:
+		return encodeCreateViewMessage(mt)
+	case *DeleteViewMessage:
+		return encodeDeleteViewMessage(mt)
+	case *ClusterStatus:
+		return encodeClusterStatus(mt)
+	case *ResizeInstruction:
+		return encodeResizeInstruction(mt)
+	case *ResizeInstructionComplete:
+		return encodeResizeInstructionComplete(mt)
+	case *SetCoordinatorMessage:
+		return encodeSetCoordinatorMessage(mt)
+	case *UpdateCoordinatorMessage:
+		return encodeUpdateCoordinatorMessage(mt)
+	case *NodeStateMessage:
+		return encodeNodeStateMessage(mt)
+	case *RecalculateCaches:
+		return encodeRecalculateCaches(mt)
+	case *nodeEvent:
+		return encodeNodeEventMessage(mt)
+	case *NodeStatus:
+		return encodeNodeStatus(mt)
 	}
 	return nil
 }
 
 // UnmarshalMessage decodes the byte slice into a protobuf message.
-func UnmarshalMessage(buf []byte) (Message, error) {
+func UnmarshalMessage(buf []byte) (proto.Message, error) {
 	typ, buf := buf[0], buf[1:]
 	var m proto.Message
 	switch typ {
@@ -177,3 +202,41 @@ func UnmarshalMessage(buf []byte) (Message, error) {
 	}
 	return m, nil
 }
+
+func decode(m proto.Message) Message {
+	switch mt := m.(type) {
+	case *internal.CreateShardMessage:
+		return decodeCreateShardMessage(mt)
+	case *internal.CreateIndexMessage:
+		return decodeCreateIndexMessage(mt)
+	case *internal.DeleteIndexMessage:
+		return decodeDeleteIndexMessage(mt)
+	case *internal.CreateFieldMessage:
+		return decodeCreateFieldMessage(mt)
+	case *internal.DeleteFieldMessage:
+		return decodeDeleteFieldMessage(mt)
+	case *internal.CreateViewMessage:
+		return decodeCreateViewMessage(mt)
+	case *internal.DeleteViewMessage:
+		return decodeDeleteViewMessage(mt)
+	case *internal.ClusterStatus:
+		return decodeClusterStatus(mt)
+	case *internal.ResizeInstruction:
+		return decodeResizeInstruction(mt)
+	case *internal.ResizeInstructionComplete:
+		return decodeResizeInstructionComplete(mt)
+	case *internal.SetCoordinatorMessage:
+		return decodeSetCoordinatorMessage(mt)
+	case *internal.UpdateCoordinatorMessage:
+		return decodeUpdateCoordinatorMessage(mt)
+	case *internal.NodeStateMessage:
+		return decodeNodeStateMessage(mt)
+	case *internal.RecalculateCaches:
+		return decodeRecalculateCaches(mt)
+	case *internal.NodeEventMessage:
+		return decodeNodeEventMessage(mt)
+	case *internal.NodeStatus:
+		return decodeNodeStatus(mt)
+	}
+	return nil
+}
diff --git a/cluster.go b/cluster.go
index fcfb08d73..e97ba46e4 100644
--- a/cluster.go
+++ b/cluster.go
@@ -268,8 +268,8 @@ func (c *cluster) setCoordinator(n *Node) error {
 	c.mu.Unlock()
 	// Send the update coordinator message to all nodes.
 	err := c.broadcaster.SendSync(
-		&internal.UpdateCoordinatorMessage{
-			New: EncodeNode(n),
+		&UpdateCoordinatorMessage{
+			New: n,
 		})
 	if err != nil {
 		return fmt.Errorf("problem sending UpdateCoordinator message: %v", err)
@@ -423,7 +423,7 @@ func (c *cluster) setNodeState(state string) error {
 	}
 
 	// Send node state to coordinator.
-	ns := &internal.NodeStateMessage{
+	ns := &NodeStateMessage{
 		NodeID: c.Node.ID,
 		State:  state,
 	}
@@ -460,12 +460,12 @@ func (c *cluster) receiveNodeState(nodeID string, state string) error {
 	return nil
 }
 
-// Status returns the internal ClusterStatus representation.
-func (c *cluster) Status() *internal.ClusterStatus {
-	return &internal.ClusterStatus{
+// Status returns the the cluster's status including what nodes it contains, it's ID, and current state.
+func (c *cluster) Status() *ClusterStatus {
+	return &ClusterStatus{
 		ClusterID: c.id,
 		State:     c.state,
-		Nodes:     EncodeNodes(c.Nodes),
+		Nodes:     c.Nodes,
 	}
 }
 
@@ -640,8 +640,8 @@ func (c *cluster) diff(other *cluster) (action string, nodeID string, err error)
 
 // fragSources returns a list of ResizeSources - for each node in the `to` cluster -
 // required to move from cluster `c` to cluster `to`.
-func (c *cluster) fragSources(to *cluster, idx *Index) (map[string][]*internal.ResizeSource, error) {
-	m := make(map[string][]*internal.ResizeSource)
+func (c *cluster) fragSources(to *cluster, idx *Index) (map[string][]*ResizeSource, error) {
+	m := make(map[string][]*ResizeSource)
 
 	// Determine if a node is being added or removed.
 	action, diffNodeID, err := c.diff(to)
@@ -700,7 +700,7 @@ func (c *cluster) fragSources(to *cluster, idx *Index) (map[string][]*internal.R
 
 	// Get the ResizeSource for each diff.
 	for nodeID, diff := range diffs {
-		m[nodeID] = []*internal.ResizeSource{}
+		m[nodeID] = []*ResizeSource{}
 		for _, frag := range diff {
 			// If there is no valid source node ID for a fragment,
 			// it likely means that the replica factor was not
@@ -711,8 +711,8 @@ func (c *cluster) fragSources(to *cluster, idx *Index) (map[string][]*internal.R
 				return nil, errors.New("not enough data to perform resize (replica factor may need to be increased)")
 			}
 
-			src := &internal.ResizeSource{
-				Node:  EncodeNode(c.unprotectedNodeByID(srcNodeID)),
+			src := &ResizeSource{
+				Node:  c.unprotectedNodeByID(srcNodeID),
 				Index: idx.Name(),
 				Field: frag.field,
 				View:  frag.view,
@@ -856,9 +856,9 @@ func (c *cluster) waitForStarted() error {
 		// TODO: Because the normal code path already sends a NodeJoin event (via
 		// memberlist), this it a bit redundant in most cases. Perhaps determine
 		// that the node has been restarted and don't do this step.
-		msg := &internal.NodeEventMessage{
-			Event: uint32(NodeJoin),
-			Node:  EncodeNode(c.Node),
+		msg := &nodeEvent{
+			Event: NodeJoin,
+			Node:  c.Node,
 		}
 		if err := c.broadcaster.SendSync(msg); err != nil {
 			return fmt.Errorf("sending restart NodeJoin: %v", err)
@@ -968,8 +968,8 @@ func (c *cluster) setStateAndBroadcast(state string) error {
 	return c.broadcaster.SendSync(c.Status())
 }
 
-func (c *cluster) sendTo(node *Node, msg proto.Message) error {
-	if err := c.broadcaster.SendTo(node, msg); err != nil {
+func (c *cluster) sendTo(node *Node, m Message) error {
+	if err := c.broadcaster.SendTo(node, m); err != nil {
 		return errors.Wrap(err, "sending")
 	}
 	return nil
@@ -1075,7 +1075,7 @@ func (c *cluster) generateResizeJobByAction(nodeAction nodeAction) (*resizeJob,
 	}
 
 	// multiIndex is a map of sources initialized with all the nodes in toCluster.
-	multiIndex := make(map[string][]*internal.ResizeSource)
+	multiIndex := make(map[string][]*ResizeSource)
 
 	for _, n := range toCluster.Nodes {
 		multiIndex[n.ID] = nil
@@ -1099,12 +1099,12 @@ func (c *cluster) generateResizeJobByAction(nodeAction nodeAction) (*resizeJob,
 			j.IDs[id] = true
 			continue
 		}
-		instr := &internal.ResizeInstruction{
+		instr := &ResizeInstruction{
 			JobID:         j.ID,
-			Node:          EncodeNode(toCluster.unprotectedNodeByID(id)),
-			Coordinator:   EncodeNode(c.coordinatorNode()),
+			Node:          toCluster.unprotectedNodeByID(id),
+			Coordinator:   c.coordinatorNode(),
 			Sources:       sources,
-			Schema:        c.holder.encodeSchema(), // Include the schema to ensure it's in sync on the receiving node.
+			Schema:        &Schema{Indexes: c.holder.Schema()}, // Include the schema to ensure it's in sync on the receiving node.
 			ClusterStatus: c.Status(),
 		}
 		j.Instructions = append(j.Instructions, instr)
@@ -1148,7 +1148,7 @@ func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error {
 		<-c.holder.opened
 
 		// Prepare the return message.
-		complete := &internal.ResizeInstructionComplete{
+		complete := &ResizeInstructionComplete{
 			JobID: instr.JobID,
 			Node:  instr.Node,
 			Error: "",
@@ -1167,7 +1167,7 @@ func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error {
 			for _, src := range instr.Sources {
 				c.logger.Printf("get shard %d for index %s from host %s", src.Shard, src.Index, src.Node.URI)
 
-				srcURI := decodeURI(src.Node.URI)
+				srcURI := src.Node.URI
 
 				// Retrieve field.
 				f := c.holder.Field(src.Index, src.Field)
@@ -1219,14 +1219,14 @@ func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error {
 			complete.Error = err.Error()
 		}
 
-		if err := c.sendTo(DecodeNode(instr.Coordinator), complete); err != nil {
+		if err := c.sendTo(instr.Coordinator, complete); err != nil {
 			c.logger.Printf("sending resizeInstructionComplete error: err=%s", err)
 		}
 	}()
 	return nil
 }
 
-func (c *cluster) markResizeInstructionComplete(complete *internal.ResizeInstructionComplete) error {
+func (c *cluster) markResizeInstructionComplete(complete *ResizeInstructionComplete) error {
 
 	j := c.job(complete.JobID)
 
@@ -1263,7 +1263,7 @@ func (c *cluster) job(id int64) *resizeJob {
 type resizeJob struct {
 	ID           int64
 	IDs          map[string]bool
-	Instructions []*internal.ResizeInstruction
+	Instructions []*ResizeInstruction
 	Broadcaster  broadcaster
 
 	action string
@@ -1366,7 +1366,7 @@ func (j *resizeJob) distributeResizeInstructions() error {
 		// a dummy node object to use in the SendTo() method.
 		node := &Node{
 			ID:  instr.Node.ID,
-			URI: decodeURI(instr.Node.URI),
+			URI: instr.Node.URI,
 		}
 		j.Logger.Printf("send resize instructions: %v", instr)
 		if err := j.Broadcaster.SendTo(node, instr); err != nil {
@@ -1757,7 +1757,7 @@ type ResizeInstruction struct {
 	ClusterStatus *ClusterStatus
 }
 
-func decodeResizeInstruction(ri *internal.ResizeInstruction) ResizeInstruction {
+func decodeResizeInstruction(ri *internal.ResizeInstruction) *ResizeInstruction {
 	return &ResizeInstruction{
 		JobID:         ri.JobID,
 		Node:          DecodeNode(ri.Node),
@@ -1768,6 +1768,17 @@ func decodeResizeInstruction(ri *internal.ResizeInstruction) ResizeInstruction {
 	}
 }
 
+func encodeResizeInstruction(m *ResizeInstruction) *internal.ResizeInstruction {
+	return &internal.ResizeInstruction{
+		JobID:         m.JobID,
+		Node:          EncodeNode(m.Node),
+		Coordinator:   EncodeNode(m.Coordinator),
+		Sources:       encodeResizeSources(m.Sources),
+		Schema:        encodeSchema(m.Schema),
+		ClusterStatus: encodeClusterStatus(m.ClusterStatus),
+	}
+}
+
 type ResizeSource struct {
 	Node  *Node  `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"`
 	Index string `protobuf:"bytes,2,opt,name=Index,proto3" json:"Index,omitempty"`
@@ -1784,6 +1795,14 @@ func decodeResizeSources(srcs []*internal.ResizeSource) []*ResizeSource {
 	return new
 }
 
+func encodeResizeSources(srcs []*ResizeSource) []*internal.ResizeSource {
+	new := make([]*internal.ResizeSource, 0, len(srcs))
+	for _, src := range srcs {
+		new = append(new, encodeResizeSource(src))
+	}
+	return new
+}
+
 func decodeResizeSource(rs *internal.ResizeSource) *ResizeSource {
 	return &ResizeSource{
 		Node:  DecodeNode(rs.Node),
@@ -1794,6 +1813,16 @@ func decodeResizeSource(rs *internal.ResizeSource) *ResizeSource {
 	}
 }
 
+func encodeResizeSource(m *ResizeSource) *internal.ResizeSource {
+	return &internal.ResizeSource{
+		Node:  EncodeNode(m.Node),
+		Index: m.Index,
+		Field: m.Field,
+		View:  m.View,
+		Shard: m.Shard,
+	}
+}
+
 // Schema is a schema
 type Schema struct {
 	Indexes []*IndexInfo
@@ -1805,6 +1834,12 @@ func decodeSchema(s *internal.Schema) *Schema {
 	}
 }
 
+func encodeSchema(m *Schema) *internal.Schema {
+	return &internal.Schema{
+		Indexes: encodeIndexInfos(m.Indexes),
+	}
+}
+
 func decodeIndexes(idxs []*internal.Index) []*IndexInfo {
 	new := make([]*IndexInfo, 0, len(idxs))
 	for _, idx := range idxs {
@@ -1813,6 +1848,14 @@ func decodeIndexes(idxs []*internal.Index) []*IndexInfo {
 	return new
 }
 
+func encodeIndexInfos(idxs []*IndexInfo) []*internal.Index {
+	new := make([]*internal.Index, 0, len(idxs))
+	for _, idx := range idxs {
+		new = append(new, encodeIndexInfo(idx))
+	}
+	return new
+}
+
 func decodeIndex(idx *internal.Index) *IndexInfo {
 	return &IndexInfo{
 		Name:   idx.Name,
@@ -1820,6 +1863,13 @@ func decodeIndex(idx *internal.Index) *IndexInfo {
 	}
 }
 
+func encodeIndexInfo(idx *IndexInfo) *internal.Index {
+	return &internal.Index{
+		Name:   idx.Name,
+		Fields: encodeFieldInfos(idx.Fields),
+	}
+}
+
 func decodeFields(fs []*internal.Field) []*FieldInfo {
 	new := make([]*FieldInfo, 0, len(fs))
 	for _, f := range fs {
@@ -1828,6 +1878,14 @@ func decodeFields(fs []*internal.Field) []*FieldInfo {
 	return new
 }
 
+func encodeFieldInfos(fs []*FieldInfo) []*internal.Field {
+	new := make([]*internal.Field, 0, len(fs))
+	for _, f := range fs {
+		new = append(new, encodeFieldInfo(f))
+	}
+	return new
+}
+
 func decodeField(f *internal.Field) *FieldInfo {
 	fi := &FieldInfo{
 		Name:    f.Name,
@@ -1840,6 +1898,19 @@ func decodeField(f *internal.Field) *FieldInfo {
 	return fi
 }
 
+func encodeFieldInfo(f *FieldInfo) *internal.Field {
+	ifield := &internal.Field{
+		Name:  f.Name,
+		Meta:  encodeFieldOptions(&f.Options),
+		Views: make([]string, 0, len(f.Views)),
+	}
+
+	for _, viewinfo := range f.Views {
+		ifield.Views = append(ifield.Views, viewinfo.Name)
+	}
+	return ifield
+}
+
 // EncodeNodes converts a slice of Nodes into its internal representation.
 func EncodeNodes(a []*Node) []*internal.Node {
 	other := make([]*internal.Node, len(a))
@@ -1878,6 +1949,14 @@ func decodeClusterStatus(cs *internal.ClusterStatus) *ClusterStatus {
 	}
 }
 
+func encodeClusterStatus(m *ClusterStatus) *internal.ClusterStatus {
+	return &internal.ClusterStatus{
+		State:     m.State,
+		ClusterID: m.ClusterID,
+		Nodes:     EncodeNodes(m.Nodes),
+	}
+}
+
 // DecodeNode converts a proto message into a Node.
 func DecodeNode(node *internal.Node) *Node {
 	return &Node{
@@ -1969,3 +2048,223 @@ func decodeIndexMeta(pb *internal.IndexMeta) *IndexOptions {
 		Keys: pb.Keys,
 	}
 }
+
+type DeleteIndexMessage struct {
+	Index string
+}
+
+func encodeDeleteIndexMessage(m *DeleteIndexMessage) *internal.DeleteIndexMessage {
+	return &internal.DeleteIndexMessage{
+		Index: m.Index,
+	}
+}
+
+func decodeDeleteIndexMessage(pb *internal.DeleteIndexMessage) *DeleteIndexMessage {
+	return &DeleteIndexMessage{
+		Index: pb.Index,
+	}
+}
+
+type CreateFieldMessage struct {
+	Index string
+	Field string
+	Meta  *FieldOptions
+}
+
+func encodeCreateFieldMessage(m *CreateFieldMessage) *internal.CreateFieldMessage {
+	return &internal.CreateFieldMessage{
+		Index: m.Index,
+		Field: m.Field,
+		Meta:  encodeFieldOptions(m.Meta),
+	}
+}
+
+func decodeCreateFieldMessage(pb *internal.CreateFieldMessage) *CreateFieldMessage {
+	return &CreateFieldMessage{
+		Index: pb.Index,
+		Field: pb.Field,
+		Meta:  decodeFieldOptions(pb.Meta),
+	}
+}
+
+type DeleteFieldMessage struct {
+	Index string
+	Field string
+}
+
+func encodeDeleteFieldMessage(m *DeleteFieldMessage) *internal.DeleteFieldMessage {
+	return &internal.DeleteFieldMessage{
+		Index: m.Index,
+		Field: m.Field,
+	}
+}
+
+func decodeDeleteFieldMessage(pb *internal.DeleteFieldMessage) *DeleteFieldMessage {
+	return &DeleteFieldMessage{
+		Index: pb.Index,
+		Field: pb.Field,
+	}
+}
+
+type CreateViewMessage struct {
+	Index string
+	Field string
+	View  string
+}
+
+func encodeCreateViewMessage(m *CreateViewMessage) *internal.CreateViewMessage {
+	return &internal.CreateViewMessage{
+		Index: m.Index,
+		Field: m.Field,
+		View:  m.View,
+	}
+}
+
+func decodeCreateViewMessage(pb *internal.CreateViewMessage) *CreateViewMessage {
+	return &CreateViewMessage{
+		Index: pb.Index,
+		Field: pb.Field,
+		View:  pb.View,
+	}
+}
+
+type DeleteViewMessage struct {
+	Index string
+	Field string
+	View  string
+}
+
+func encodeDeleteViewMessage(m *DeleteViewMessage) *internal.DeleteViewMessage {
+	return &internal.DeleteViewMessage{
+		Index: m.Index,
+		Field: m.Field,
+		View:  m.View,
+	}
+}
+
+func decodeDeleteViewMessage(pb *internal.DeleteViewMessage) *DeleteViewMessage {
+	return &DeleteViewMessage{
+		Index: pb.Index,
+		Field: pb.Field,
+		View:  pb.View,
+	}
+}
+
+type ResizeInstructionComplete struct {
+	JobID int64
+	Node  *Node
+	Error string
+}
+
+func encodeResizeInstructionComplete(m *ResizeInstructionComplete) *internal.ResizeInstructionComplete {
+	return &internal.ResizeInstructionComplete{
+		JobID: m.JobID,
+		Node:  EncodeNode(m.Node),
+		Error: m.Error,
+	}
+}
+
+func decodeResizeInstructionComplete(pb *internal.ResizeInstructionComplete) *ResizeInstructionComplete {
+	return &ResizeInstructionComplete{
+		JobID: pb.JobID,
+		Node:  DecodeNode(pb.Node),
+		Error: pb.Error,
+	}
+}
+
+type SetCoordinatorMessage struct {
+	New *Node
+}
+
+func encodeSetCoordinatorMessage(m *SetCoordinatorMessage) *internal.SetCoordinatorMessage {
+	return &internal.SetCoordinatorMessage{
+		New: EncodeNode(m.New),
+	}
+}
+
+func decodeSetCoordinatorMessage(pb *internal.SetCoordinatorMessage) *SetCoordinatorMessage {
+	return &SetCoordinatorMessage{
+		New: DecodeNode(pb.New),
+	}
+}
+
+type UpdateCoordinatorMessage struct {
+	New *Node
+}
+
+func encodeUpdateCoordinatorMessage(m *UpdateCoordinatorMessage) *internal.UpdateCoordinatorMessage {
+	return &internal.UpdateCoordinatorMessage{
+		New: EncodeNode(m.New),
+	}
+}
+
+func decodeUpdateCoordinatorMessage(pb *internal.UpdateCoordinatorMessage) *UpdateCoordinatorMessage {
+	return &UpdateCoordinatorMessage{
+		New: DecodeNode(pb.New),
+	}
+}
+
+type NodeStateMessage struct {
+	NodeID string `protobuf:"bytes,1,opt,name=NodeID,proto3" json:"NodeID,omitempty"`
+	State  string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"`
+}
+
+func encodeNodeStateMessage(m *NodeStateMessage) *internal.NodeStateMessage {
+	return &internal.NodeStateMessage{
+		NodeID: m.NodeID,
+		State:  m.State,
+	}
+}
+
+func decodeNodeStateMessage(pb *internal.NodeStateMessage) *NodeStateMessage {
+	return &NodeStateMessage{
+		NodeID: pb.NodeID,
+		State:  pb.State,
+	}
+}
+
+func encodeNodeEventMessage(m *nodeEvent) *internal.NodeEventMessage {
+	return &internal.NodeEventMessage{
+		Event: uint32(m.Event),
+		Node:  EncodeNode(m.Node),
+	}
+}
+
+func decodeNodeEventMessage(pb *internal.NodeEventMessage) *nodeEvent {
+	return &nodeEvent{
+		Event: NodeEventType(pb.Event),
+		Node:  DecodeNode(pb.Node),
+	}
+}
+
+type NodeStatus struct {
+	Node      *Node
+	MaxShards map[string]uint64
+	Schema    *Schema
+}
+
+func encodeNodeStatus(m *NodeStatus) *internal.NodeStatus {
+	return &internal.NodeStatus{
+		Node:      EncodeNode(m.Node),
+		MaxShards: &internal.MaxShards{Standard: m.MaxShards},
+		Schema:    encodeSchema(m.Schema),
+	}
+}
+
+func decodeNodeStatus(pb *internal.NodeStatus) *NodeStatus {
+	return &NodeStatus{
+		Node:      DecodeNode(pb.Node),
+		MaxShards: pb.MaxShards.Standard,
+		Schema:    decodeSchema(pb.Schema),
+	}
+}
+
+type RecalculateCaches struct{}
+
+func decodeRecalculateCaches(pb *internal.RecalculateCaches) *RecalculateCaches {
+	return &RecalculateCaches{}
+}
+
+func encodeRecalculateCaches(*RecalculateCaches) *internal.RecalculateCaches {
+	return &internal.RecalculateCaches{}
+}
diff --git a/holder.go b/holder.go
index e599d21f2..83ffd6967 100644
--- a/holder.go
+++ b/holder.go
@@ -230,7 +230,7 @@ func (h *Holder) Schema() []*IndexInfo {
 }
 
 // applySchema applies an internal Schema to Holder.
-func (h *Holder) applySchema(schema *internal.Schema) error {
+func (h *Holder) applySchema(schema *Schema) error {
 	// Create indexes that don't exist.
 	for _, index := range schema.Indexes {
 		opt := IndexOptions{}
@@ -240,14 +240,13 @@ func (h *Holder) applySchema(schema *internal.Schema) error {
 		}
 		// Create fields that don't exist.
 		for _, f := range index.Fields {
-			opt := decodeFieldOptions(f.Meta)
-			field, err := idx.CreateFieldIfNotExists(f.Name, *opt)
+			field, err := idx.CreateFieldIfNotExists(f.Name, f.Options)
 			if err != nil {
 				return errors.Wrap(err, "creating field")
 			}
 			// Create views that don't exist.
 			for _, v := range f.Views {
-				_, err := field.createViewIfNotExists(v)
+				_, err := field.createViewIfNotExists(v.Name)
 				if err != nil {
 					return errors.Wrap(err, "creating view")
 				}
diff --git a/server.go b/server.go
index dffeb94a0..7199b7178 100644
--- a/server.go
+++ b/server.go
@@ -27,8 +27,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/gogo/protobuf/proto"
-	"github.com/pilosa/pilosa/internal"
 	"github.com/pkg/errors"
 
 	"golang.org/x/sync/errgroup"
@@ -431,40 +429,40 @@ func (s *Server) monitorAntiEntropy() {
 }
 
 // receiveMessage represents an implementation of BroadcastHandler.
-func (s *Server) receiveMessage(pb proto.Message) error {
-	switch obj := pb.(type) {
-	case *internal.CreateShardMessage:
+func (s *Server) receiveMessage(m Message) error {
+	switch obj := m.(type) {
+	case *CreateShardMessage:
 		idx := s.holder.Index(obj.Index)
 		if idx == nil {
 			return fmt.Errorf("Local Index not found: %s", obj.Index)
 		}
 		idx.setRemoteMaxShard(obj.Shard)
-	case *internal.CreateIndexMessage:
+	case *CreateIndexMessage:
 		opt := IndexOptions{}
 		_, err := s.holder.CreateIndex(obj.Index, opt)
 		if err != nil {
 			return err
 		}
-	case *internal.DeleteIndexMessage:
+	case *DeleteIndexMessage:
 		if err := s.holder.DeleteIndex(obj.Index); err != nil {
 			return err
 		}
-	case *internal.CreateFieldMessage:
+	case *CreateFieldMessage:
 		idx := s.holder.Index(obj.Index)
 		if idx == nil {
 			return fmt.Errorf("Local Index not found: %s", obj.Index)
 		}
-		opt := decodeFieldOptions(obj.Meta)
+		opt := obj.Meta
 		_, err := idx.CreateField(obj.Field, *opt)
 		if err != nil {
 			return err
 		}
-	case *internal.DeleteFieldMessage:
+	case *DeleteFieldMessage:
 		idx := s.holder.Index(obj.Index)
 		if err := idx.DeleteField(obj.Field); err != nil {
 			return err
 		}
-	case *internal.CreateViewMessage:
+	case *CreateViewMessage:
 		f := s.holder.Field(obj.Index, obj.Field)
 		if f == nil {
 			return fmt.Errorf("Local Field not found: %s", obj.Field)
@@ -473,7 +471,7 @@ func (s *Server) receiveMessage(pb proto.Message) error {
 		if err != nil {
 			return err
 		}
-	case *internal.DeleteViewMessage:
+	case *DeleteViewMessage:
 		f := s.holder.Field(obj.Index, obj.Field)
 		if f == nil {
 			return fmt.Errorf("Local Field not found: %s", obj.Field)
@@ -482,36 +480,36 @@ func (s *Server) receiveMessage(pb proto.Message) error {
 		if err != nil {
 			return err
 		}
-	case *internal.ClusterStatus:
-		err := s.cluster.mergeClusterStatus(decodeClusterStatus(obj))
+	case *ClusterStatus:
+		err := s.cluster.mergeClusterStatus(obj)
 		if err != nil {
 			return err
 		}
-	case *internal.ResizeInstruction:
-		err := s.cluster.followResizeInstruction(decodeResizeInstruction(obj))
+	case *ResizeInstruction:
+		err := s.cluster.followResizeInstruction(obj)
 		if err != nil {
 			return err
 		}
-	case *internal.ResizeInstructionComplete:
+	case *ResizeInstructionComplete:
 		err := s.cluster.markResizeInstructionComplete(obj)
 		if err != nil {
 			return err
 		}
-	case *internal.SetCoordinatorMessage:
-		s.cluster.setCoordinator(DecodeNode(obj.New))
-	case *internal.UpdateCoordinatorMessage:
-		s.cluster.updateCoordinator(DecodeNode(obj.New))
-	case *internal.NodeStateMessage:
+	case *SetCoordinatorMessage:
+		s.cluster.setCoordinator(obj.New)
+	case *UpdateCoordinatorMessage:
+		s.cluster.updateCoordinator(obj.New)
+	case *NodeStateMessage:
 		err := s.cluster.receiveNodeState(obj.NodeID, obj.State)
 		if err != nil {
 			return err
 		}
-	case *internal.RecalculateCaches:
+	case *RecalculateCaches:
 		s.holder.RecalculateCaches()
-	case *internal.NodeEventMessage:
-		s.cluster.ReceiveEvent(DecodeNodeEvent(obj))
-	case *internal.NodeStatus:
-		s.handleRemoteStatus(pb)
+	case *nodeEvent:
+		s.cluster.ReceiveEvent(obj)
+	case *NodeStatus:
+		s.handleRemoteStatus(obj)
 	}
 
 	return nil
@@ -519,6 +517,7 @@ func (s *Server) receiveMessage(pb proto.Message) error {
 
 // SendSync represents an implementation of Broadcaster.
 func (s *Server) SendSync(m Message) error {
+	pb := encode(m)
 	var eg errgroup.Group
 	for _, node := range s.cluster.Nodes {
 		node := node
@@ -537,12 +536,13 @@ func (s *Server) SendSync(m Message) error {
 }
 
 // SendAsync represents an implementation of Broadcaster.
-func (s *Server) SendAsync(pb proto.Message) error {
+func (s *Server) SendAsync(m Message) error {
 	return ErrNotImplemented
 }
 
 // SendTo represents an implementation of Broadcaster.
-func (s *Server) SendTo(to *Node, pb proto.Message) error {
+func (s *Server) SendTo(to *Node, m Message) error {
+	pb := encode(m)
 	s.logger.Printf("SendTo: %s", to.URI)
 	return s.defaultClient.SendMessage(context.Background(), &to.URI, pb)
 }
@@ -554,7 +554,7 @@ func (s *Server) node() Node {
 }
 
 // handleRemoteStatus receives incoming NodeStatus from remote nodes.
-func (s *Server) handleRemoteStatus(pb proto.Message) {
+func (s *Server) handleRemoteStatus(pb Message) {
 	// Ignore NodeStatus messages until the cluster is in a Normal state.
 	if s.cluster.State() != ClusterStateNormal {
 		return
@@ -564,16 +564,16 @@ func (s *Server) handleRemoteStatus(pb proto.Message) {
 		// Make sure the holder has opened.
 		<-s.holder.opened
 
-		err := s.mergeRemoteStatus(pb.(*internal.NodeStatus))
+		err := s.mergeRemoteStatus(pb.(*NodeStatus))
 		if err != nil {
 			s.logger.Printf("merge remote status: %s", err)
 		}
 	}()
 }
 
-func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error {
+func (s *Server) mergeRemoteStatus(ns *NodeStatus) error {
 	// Ignore status updates from self.
-	if s.nodeID == DecodeNode(ns.Node).ID {
+	if s.nodeID == ns.Node.ID {
 		return nil
 	}
 
@@ -584,7 +584,7 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error {
 
 	// Sync maxShards.
 	oldmaxshards := s.holder.maxShards()
-	for index, newMax := range ns.MaxShards.Standard {
+	for index, newMax := range ns.MaxShards {
 		localIndex := s.holder.Index(index)
 		// if we don't know about an index locally, log an error because
 		// indexes should be created and synced prior to shard creation
diff --git a/utils_internal_test.go b/utils_internal_test.go
index ca7bd1fa7..f7c11d6a7 100644
--- a/utils_internal_test.go
+++ b/utils_internal_test.go
@@ -24,7 +24,6 @@ import (
 	"time"
 
 	"github.com/gogo/protobuf/proto"
-	"github.com/pilosa/pilosa/internal"
 )
 
 // NewTestCluster returns a cluster with n nodes and uses a mod-based hasher.
@@ -304,9 +303,9 @@ func (t *ClusterCluster) Close() error {
 }
 
 // SendSync is a test implemenetation of Broadcaster SendSync method.
-func (t *ClusterCluster) SendSync(pb proto.Message) error {
-	switch obj := pb.(type) {
-	case *internal.ClusterStatus:
+func (t *ClusterCluster) SendSync(m Message) error {
+	switch obj := m.(type) {
+	case *ClusterStatus:
 		// Apply the send message to all nodes (except the coordinator).
 		for _, c := range t.Clusters {
 			c.mergeClusterStatus(obj)
@@ -322,19 +321,19 @@ func (t *ClusterCluster) SendSync(pb proto.Message) error {
 }
 
 // SendAsync is a test implemenetation of Broadcaster SendAsync method.
-func (t *ClusterCluster) SendAsync(pb proto.Message) error {
+func (t *ClusterCluster) SendAsync(Message) error {
 	return nil
 }
 
 // SendTo is a test implemenetation of Broadcaster SendTo method.
-func (t *ClusterCluster) SendTo(to *Node, pb proto.Message) error {
-	switch obj := pb.(type) {
-	case *internal.ResizeInstruction:
+func (t *ClusterCluster) SendTo(to *Node, m Message) error {
+	switch obj := m.(type) {
+	case *ResizeInstruction:
 		err := t.FollowResizeInstruction(obj)
 		if err != nil {
 			return err
 		}
-	case *internal.ResizeInstructionComplete:
+	case *ResizeInstructionComplete:
 		coord := t.clusterByID(to.ID)
 		go coord.markResizeInstructionComplete(obj)
 	}
@@ -342,10 +341,10 @@ func (t *ClusterCluster) SendTo(to *Node, pb proto.Message) error {
 }
 
 // FollowResizeInstruction is a version of cluster.FollowResizeInstruction used for testing.
-func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstruction) error {
+func (t *ClusterCluster) FollowResizeInstruction(instr *ResizeInstruction) error {
 
 	// Prepare the return message.
-	complete := &internal.ResizeInstructionComplete{
+	complete := &ResizeInstructionComplete{
 		JobID: instr.JobID,
 		Node:  instr.Node,
 		Error: "",
@@ -356,7 +355,7 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstructi
 
 		// figure out which node it was meant for, then call the operation on that cluster
 		// basically need to mimic this: client.RetrieveShardFromURI(context.Background(), src.Index, src.Field, src.View, src.Shard, srcURI)
-		instrNode := DecodeNode(instr.Node)
+		instrNode := instr.Node
 		destCluster := t.clusterByID(instrNode.ID)
 
 		// Sync the schema received in the resize instruction.
@@ -365,7 +364,7 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstructi
 		}
 
 		for _, src := range instr.Sources {
-			srcNode := DecodeNode(src.Node)
+			srcNode := src.Node
 			srcCluster := t.clusterByID(srcNode.ID)
 
 			srcFragment := srcCluster.holder.fragment(src.Index, src.Field, src.View, src.Shard)
@@ -405,6 +404,6 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstructi
 		complete.Error = err.Error()
 	}
 
-	node := DecodeNode(instr.Coordinator)
+	node := instr.Coordinator
 	return t.SendTo(node, complete)
 }

From 44f0b992f6170efc2d802be99942b1a71f25f8ee Mon Sep 17 00:00:00 2001
From: Travis Turner <travis@pilosa.com>
Date: Wed, 4 Jul 2018 21:22:35 -0500
Subject: [PATCH 09/32] change all CreateField() methods to take functional
 options instead of FieldOptions

---
 api.go                    |   2 +-
 cluster_internal_test.go  |   4 +-
 executor_test.go          | 102 +++++++++++---------------------------
 field.go                  |  30 ++++++++++-
 field_internal_test.go    |  18 +++----
 field_test.go             |  34 +++----------
 fragment_internal_test.go |   4 +-
 holder.go                 |   2 +-
 holder_internal_test.go   |   8 +--
 holder_test.go            |  10 ++--
 http/client_test.go       |   7 +--
 index.go                  |  37 ++++++++++++--
 index_test.go             |  17 ++-----
 server.go                 |   2 +-
 server/handler_test.go    |  10 ++--
 test/field.go             |  12 ++---
 test/holder.go            |  12 ++---
 test/index.go             |   8 +--
 utils_internal_test.go    |   4 +-
 19 files changed, 151 insertions(+), 172 deletions(-)

diff --git a/api.go b/api.go
index abf2a84ea..f0e2dbfb0 100644
--- a/api.go
+++ b/api.go
@@ -257,7 +257,7 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str
 	}
 
 	// Create field.
-	field, err := index.CreateField(fieldName, fo)
+	field, err := index.CreateField(fieldName, opts)
 	if err != nil {
 		return nil, errors.Wrap(err, "creating field")
 	}
diff --git a/cluster_internal_test.go b/cluster_internal_test.go
index aee4ea09b..6dc79fdef 100644
--- a/cluster_internal_test.go
+++ b/cluster_internal_test.go
@@ -150,7 +150,7 @@ func TestFragSources(t *testing.T) {
 	c5.addNodeBasicSorted(node3)
 
 	idx := newIndexWithTempPath("i")
-	field, err := idx.CreateFieldIfNotExists("f", FieldOptions{})
+	field, err := idx.CreateFieldIfNotExists("f", OptFieldTypeDefault())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -697,7 +697,7 @@ func TestCluster_ResizeStates(t *testing.T) {
 		}
 
 		// Add Bit Data to node0.
-		if err := tc.CreateField("i", "f", FieldOptions{}); err != nil {
+		if err := tc.CreateField("i", "f", OptFieldTypeDefault()); err != nil {
 			t.Fatal(err)
 		}
 		tc.SetBit("i", "f", 1, 101, nil)
diff --git a/executor_test.go b/executor_test.go
index f54fb7e33..c5e2f24a8 100644
--- a/executor_test.go
+++ b/executor_test.go
@@ -39,7 +39,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
 		defer c.Close()
 		hldr := test.Holder{Holder: c[0].Server.Holder()}
 		index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
-		f, err := index.CreateField("f", pilosa.FieldOptions{})
+		f, err := index.CreateField("f", pilosa.OptFieldTypeDefault())
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -89,7 +89,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
 		hldr := test.Holder{Holder: c[0].Server.Holder()}
 
 		index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
-		if _, err := index.CreateField("f", pilosa.FieldOptions{}); err != nil {
+		if _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil {
 			t.Fatal(err)
 		}
 
@@ -112,7 +112,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
 		hldr := test.Holder{Holder: c[0].Server.Holder()}
 
 		index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true})
-		if _, err := index.CreateField("f", pilosa.FieldOptions{Keys: true}); err != nil {
+		if _, err := index.CreateField("f", pilosa.OptFieldTypeDefault(), pilosa.OptFieldKeys()); err != nil {
 			t.Fatal(err)
 		}
 
@@ -354,7 +354,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
 			if err := index.DeleteField("f"); err != nil {
 				t.Fatal(err)
 			}
-			if _, err := index.CreateField("f", pilosa.FieldOptions{}); err != nil {
+			if _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil {
 				t.Fatal(err)
 			}
 
@@ -365,7 +365,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
 
 		t.Run("ErrInvalidRowValueType", func(t *testing.T) {
 			index := hldr.MustCreateIndexIfNotExists("inokey", pilosa.IndexOptions{})
-			if _, err := index.CreateField("f", pilosa.FieldOptions{Keys: true}); err != nil {
+			if _, err := index.CreateField("f", pilosa.OptFieldTypeDefault(), pilosa.OptFieldKeys()); err != nil {
 				t.Fatal(err)
 			}
 			if _, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "inokey", Query: `Set(2, f=1)`}); err == nil || errors.Cause(err).Error() != `row value must be a string when field 'keys' option enabled` {
@@ -398,13 +398,9 @@ func TestExecutor_Execute_SetValue(t *testing.T) {
 
 		// Create felds.
 		index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
-		if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{
-			Type: pilosa.FieldTypeInt,
-			Min:  0,
-			Max:  50,
-		}); err != nil {
+		if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeInt(0, 50)); err != nil {
 			t.Fatal(err)
-		} else if _, err := index.CreateFieldIfNotExists("xxx", pilosa.FieldOptions{}); err != nil {
+		} else if _, err := index.CreateFieldIfNotExists("xxx", pilosa.OptFieldTypeDefault()); err != nil {
 			t.Fatal(err)
 		}
 
@@ -439,11 +435,7 @@ func TestExecutor_Execute_SetValue(t *testing.T) {
 		hldr := test.Holder{Holder: c[0].Server.Holder()}
 
 		index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
-		if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{
-			Type: pilosa.FieldTypeInt,
-			Min:  0,
-			Max:  100,
-		}); err != nil {
+		if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeInt(0, 100)); err != nil {
 			t.Fatal(err)
 		}
 
@@ -475,9 +467,9 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) {
 
 	// Create fields.
 	index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
-	if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{}); err != nil {
+	if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeDefault()); err != nil {
 		t.Fatal(err)
-	} else if _, err := index.CreateFieldIfNotExists("xxx", pilosa.FieldOptions{}); err != nil {
+	} else if _, err := index.CreateFieldIfNotExists("xxx", pilosa.OptFieldTypeDefault()); err != nil {
 		t.Fatal(err)
 	}
 
@@ -514,9 +506,9 @@ func TestExecutor_Execute_TopN(t *testing.T) {
 		// Set columns for rows 0, 10, & 20 across two shards.
 		if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil {
 			t.Fatal(err)
-		} else if _, err := idx.CreateField("f", pilosa.FieldOptions{}); err != nil {
+		} else if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil {
 			t.Fatal(err)
-		} else if _, err := idx.CreateField("other", pilosa.FieldOptions{}); err != nil {
+		} else if _, err := idx.CreateField("other", pilosa.OptFieldTypeDefault()); err != nil {
 			t.Fatal(err)
 		} else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `
 			Set(0, f=0)
@@ -555,9 +547,9 @@ func TestExecutor_Execute_TopN(t *testing.T) {
 		// Set columns for rows 0, 10, & 20 across two shards.
 		if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{Keys: true}); err != nil {
 			t.Fatal(err)
-		} else if _, err := idx.CreateField("f", pilosa.FieldOptions{Keys: true}); err != nil {
+		} else if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault(), pilosa.OptFieldKeys()); err != nil {
 			t.Fatal(err)
-		} else if _, err := idx.CreateField("other", pilosa.FieldOptions{Keys: true}); err != nil {
+		} else if _, err := idx.CreateField("other", pilosa.OptFieldTypeDefault(), pilosa.OptFieldKeys()); err != nil {
 			t.Fatal(err)
 		} else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `
 			Set("a", f="foo")
@@ -741,15 +733,11 @@ func TestExecutor_Execute_MinMax(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if _, err := idx.CreateField("x", pilosa.FieldOptions{}); err != nil {
+	if _, err := idx.CreateField("x", pilosa.OptFieldTypeDefault()); err != nil {
 		t.Fatal(err)
 	}
 
-	if _, err := idx.CreateField("f", pilosa.FieldOptions{
-		Type: pilosa.FieldTypeInt,
-		Min:  -10,
-		Max:  100,
-	}); err != nil {
+	if _, err := idx.CreateField("f", pilosa.OptFieldTypeInt(-10, 100)); err != nil {
 		t.Fatal(err)
 	}
 
@@ -836,31 +824,19 @@ func TestExecutor_Execute_Sum(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if _, err := idx.CreateField("x", pilosa.FieldOptions{}); err != nil {
+	if _, err := idx.CreateField("x", pilosa.OptFieldTypeDefault()); err != nil {
 		t.Fatal(err)
 	}
 
-	if _, err := idx.CreateField("foo", pilosa.FieldOptions{
-		Type: pilosa.FieldTypeInt,
-		Min:  10,
-		Max:  100,
-	}); err != nil {
+	if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10, 100)); err != nil {
 		t.Fatal(err)
 	}
 
-	if _, err := idx.CreateField("bar", pilosa.FieldOptions{
-		Type: pilosa.FieldTypeInt,
-		Min:  0,
-		Max:  100000,
-	}); err != nil {
+	if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100000)); err != nil {
 		t.Fatal(err)
 	}
 
-	if _, err := idx.CreateField("other", pilosa.FieldOptions{
-		Type: pilosa.FieldTypeInt,
-		Min:  0,
-		Max:  1000,
-	}); err != nil {
+	if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0, 1000)); err != nil {
 		t.Fatal(err)
 	}
 
@@ -906,10 +882,7 @@ func TestExecutor_Execute_Range(t *testing.T) {
 	index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
 
 	// Create field.
-	if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{
-		Type:        pilosa.FieldTypeTime,
-		TimeQuantum: pilosa.TimeQuantum("YMDH"),
-	}); err != nil {
+	if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))); err != nil {
 		t.Fatal(err)
 	}
 
@@ -962,39 +935,23 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if _, err := idx.CreateField("f", pilosa.FieldOptions{}); err != nil {
+	if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil {
 		t.Fatal(err)
 	}
 
-	if _, err := idx.CreateField("foo", pilosa.FieldOptions{
-		Type: pilosa.FieldTypeInt,
-		Min:  10,
-		Max:  100,
-	}); err != nil {
+	if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10, 100)); err != nil {
 		t.Fatal(err)
 	}
 
-	if _, err := idx.CreateField("bar", pilosa.FieldOptions{
-		Type: pilosa.FieldTypeInt,
-		Min:  0,
-		Max:  100000,
-	}); err != nil {
+	if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100000)); err != nil {
 		t.Fatal(err)
 	}
 
-	if _, err := idx.CreateField("other", pilosa.FieldOptions{
-		Type: pilosa.FieldTypeInt,
-		Min:  0,
-		Max:  1000,
-	}); err != nil {
+	if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0, 1000)); err != nil {
 		t.Fatal(err)
 	}
 
-	if _, err := idx.CreateField("edge", pilosa.FieldOptions{
-		Type: pilosa.FieldTypeInt,
-		Min:  -100,
-		Max:  100,
-	}); err != nil {
+	if _, err := idx.CreateField("edge", pilosa.OptFieldTypeInt(-100, 100)); err != nil {
 		t.Fatal(err)
 	}
 
@@ -1268,7 +1225,7 @@ func TestExecutor_SetColumnAttrs_ExcludeField(t *testing.T) {
 	hldr := test.Holder{Holder: c[0].Server.Holder()}
 
 	index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
-	_, err := index.CreateField("f", pilosa.FieldOptions{})
+	_, err := index.CreateField("f", pilosa.OptFieldTypeDefault())
 	if err != nil {
 		t.Fatalf("creating field: %v", err)
 	}
@@ -1352,10 +1309,7 @@ func TestExecutor_Time_Clear_Quantums(t *testing.T) {
 			indexName := strings.ToLower(string(tt.quantum))
 			index := hldr.MustCreateIndexIfNotExists(indexName, pilosa.IndexOptions{})
 			// Create field.
-			if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{
-				Type:        pilosa.FieldTypeTime,
-				TimeQuantum: tt.quantum,
-			}); err != nil {
+			if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeTime(tt.quantum)); err != nil {
 				t.Fatal(err)
 			}
 			// Populate
diff --git a/field.go b/field.go
index 76be9cf3f..2b78176e1 100644
--- a/field.go
+++ b/field.go
@@ -78,6 +78,25 @@ type Field struct {
 // FieldOption is a functional option type for pilosa.FieldOptions.
 type FieldOption func(fo *FieldOptions) error
 
+func OptFieldKeys() FieldOption {
+	return func(fo *FieldOptions) error {
+		fo.Keys = true
+		return nil
+	}
+}
+
+func OptFieldTypeDefault() FieldOption {
+	return func(fo *FieldOptions) error {
+		if fo.Type != "" {
+			return errors.Errorf("field type is already set to: %s", fo.Type)
+		}
+		fo.Type = FieldTypeSet
+		fo.CacheType = DefaultCacheType
+		fo.CacheSize = DefaultCacheSize
+		return nil
+	}
+}
+
 func OptFieldTypeSet(cacheType string, cacheSize uint32) FieldOption {
 	return func(fo *FieldOptions) error {
 		if fo.Type != "" {
@@ -120,12 +139,19 @@ func OptFieldTypeTime(timeQuantum TimeQuantum) FieldOption {
 }
 
 // NewField returns a new instance of field.
-func NewField(path, index, name string, options FieldOptions) (*Field, error) {
+func NewField(path, index, name string, opts FieldOption) (*Field, error) {
 	err := validateName(name)
 	if err != nil {
 		return nil, err
 	}
 
+	// Apply functional option.
+	fo := FieldOptions{}
+	err = opts(&fo)
+	if err != nil {
+		return nil, errors.Wrap(err, "applying option")
+	}
+
 	f := &Field{
 		path:  path,
 		index: index,
@@ -138,7 +164,7 @@ func NewField(path, index, name string, options FieldOptions) (*Field, error) {
 		broadcaster: NopBroadcaster,
 		Stats:       NopStatsClient,
 
-		options: applyDefaultOptions(options),
+		options: applyDefaultOptions(fo),
 
 		Logger: NopLogger,
 	}
diff --git a/field_internal_test.go b/field_internal_test.go
index a11b5c394..4ded2bebe 100644
--- a/field_internal_test.go
+++ b/field_internal_test.go
@@ -153,7 +153,7 @@ func TestBSIGroup_BaseValue(t *testing.T) {
 
 // Ensure field can open and retrieve a view.
 func TestField_DeleteView(t *testing.T) {
-	f := MustOpenField(FieldOptions{})
+	f := MustOpenField(OptFieldTypeDefault())
 	defer f.Close()
 
 	viewName := viewStandard + "_v"
@@ -190,12 +190,12 @@ type TestField struct {
 }
 
 // NewTestField returns a new instance of TestField d/0.
-func NewTestField(options FieldOptions) *TestField {
+func NewTestField(opts FieldOption) *TestField {
 	path, err := ioutil.TempDir("", "pilosa-field-")
 	if err != nil {
 		panic(err)
 	}
-	field, err := NewField(path, "i", "f", options)
+	field, err := NewField(path, "i", "f", opts)
 	if err != nil {
 		panic(err)
 	}
@@ -203,8 +203,8 @@ func NewTestField(options FieldOptions) *TestField {
 }
 
 // MustOpenField returns a new, opened field at a temporary path. Panic on error.
-func MustOpenField(options FieldOptions) *TestField {
-	f := NewTestField(options)
+func MustOpenField(opts FieldOption) *TestField {
+	f := NewTestField(opts)
 	if err := f.Open(); err != nil {
 		panic(err)
 	}
@@ -225,7 +225,7 @@ func (f *TestField) Reopen() error {
 	}
 
 	path, index, name := f.Path(), f.Index(), f.Name()
-	f.Field, err = NewField(path, index, name, FieldOptions{})
+	f.Field, err = NewField(path, index, name, OptFieldTypeDefault())
 	if err != nil {
 		return err
 	}
@@ -253,7 +253,7 @@ func (f *TestField) MustSetBit(row, col uint64, ts ...time.Time) {
 
 // Ensure field can open and retrieve a view.
 func TestField_CreateViewIfNotExists(t *testing.T) {
-	f := MustOpenField(FieldOptions{})
+	f := MustOpenField(OptFieldTypeDefault())
 	defer f.Close()
 
 	// Create view.
@@ -278,7 +278,7 @@ func TestField_CreateViewIfNotExists(t *testing.T) {
 }
 
 func TestField_SetTimeQuantum(t *testing.T) {
-	f := MustOpenField(FieldOptions{Type: FieldTypeTime})
+	f := MustOpenField(OptFieldTypeTime(TimeQuantum("")))
 	defer f.Close()
 
 	// Set & retrieve time quantum.
@@ -297,7 +297,7 @@ func TestField_SetTimeQuantum(t *testing.T) {
 }
 
 func TestField_RowTime(t *testing.T) {
-	f := MustOpenField(FieldOptions{Type: FieldTypeTime})
+	f := MustOpenField(OptFieldTypeTime(TimeQuantum("")))
 	defer f.Close()
 
 	if err := f.SetTimeQuantum(TimeQuantum("YMDH")); err != nil {
diff --git a/field_test.go b/field_test.go
index 4b1def96c..520179f52 100644
--- a/field_test.go
+++ b/field_test.go
@@ -28,11 +28,7 @@ func TestField_SetValue(t *testing.T) {
 		idx := test.MustOpenIndex()
 		defer idx.Close()
 
-		f, err := idx.CreateField("f", pilosa.FieldOptions{
-			Type: pilosa.FieldTypeInt,
-			Min:  0,
-			Max:  30,
-		})
+		f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(0, 30))
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -65,11 +61,7 @@ func TestField_SetValue(t *testing.T) {
 		idx := test.MustOpenIndex()
 		defer idx.Close()
 
-		f, err := idx.CreateField("f", pilosa.FieldOptions{
-			Type: pilosa.FieldTypeInt,
-			Min:  0,
-			Max:  30,
-		})
+		f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(0, 30))
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -102,9 +94,7 @@ func TestField_SetValue(t *testing.T) {
 		idx := test.MustOpenIndex()
 		defer idx.Close()
 
-		f, err := idx.CreateField("f", pilosa.FieldOptions{
-			Type: pilosa.FieldTypeSet,
-		})
+		f, err := idx.CreateField("f", pilosa.OptFieldTypeDefault())
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -119,11 +109,7 @@ func TestField_SetValue(t *testing.T) {
 		idx := test.MustOpenIndex()
 		defer idx.Close()
 
-		f, err := idx.CreateField("f", pilosa.FieldOptions{
-			Type: pilosa.FieldTypeInt,
-			Min:  20,
-			Max:  30,
-		})
+		f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(20, 30))
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -138,11 +124,7 @@ func TestField_SetValue(t *testing.T) {
 		idx := test.MustOpenIndex()
 		defer idx.Close()
 
-		f, err := idx.CreateField("f", pilosa.FieldOptions{
-			Type: pilosa.FieldTypeInt,
-			Min:  20,
-			Max:  30,
-		})
+		f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(20, 30))
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -159,7 +141,7 @@ func TestField_NameRestriction(t *testing.T) {
 	if err != nil {
 		panic(err)
 	}
-	field, err := pilosa.NewField(path, "i", ".meta", pilosa.FieldOptions{})
+	field, err := pilosa.NewField(path, "i", ".meta", pilosa.OptFieldTypeDefault())
 	if field != nil {
 		t.Fatalf("unexpected field name %s", err)
 	}
@@ -191,13 +173,13 @@ func TestField_NameValidation(t *testing.T) {
 		panic(err)
 	}
 	for _, name := range validFieldNames {
-		_, err := pilosa.NewField(path, "i", name, pilosa.FieldOptions{})
+		_, err := pilosa.NewField(path, "i", name, pilosa.OptFieldTypeDefault())
 		if err != nil {
 			t.Fatalf("unexpected field name: %s %s", name, err)
 		}
 	}
 	for _, name := range invalidFieldNames {
-		_, err := pilosa.NewField(path, "i", name, pilosa.FieldOptions{})
+		_, err := pilosa.NewField(path, "i", name, pilosa.OptFieldTypeDefault())
 		if err == nil {
 			t.Fatalf("expected error on field name: %s", name)
 		}
diff --git a/fragment_internal_test.go b/fragment_internal_test.go
index eda4cbbdc..4ceb33819 100644
--- a/fragment_internal_test.go
+++ b/fragment_internal_test.go
@@ -750,7 +750,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) {
 	defer index.Close()
 
 	// Create field.
-	field, err := index.CreateFieldIfNotExists("f", FieldOptions{CacheType: CacheTypeRanked, CacheSize: cacheSize})
+	field, err := index.CreateFieldIfNotExists("f", OptFieldTypeSet(CacheTypeRanked, cacheSize))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -916,7 +916,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) {
 	defer index.Close()
 
 	// Create field.
-	field, err := index.CreateFieldIfNotExists("f", FieldOptions{CacheType: CacheTypeRanked})
+	field, err := index.CreateFieldIfNotExists("f", OptFieldTypeSet(CacheTypeRanked, DefaultCacheSize))
 	if err != nil {
 		t.Fatal(err)
 	}
diff --git a/holder.go b/holder.go
index e599d21f2..3aae91475 100644
--- a/holder.go
+++ b/holder.go
@@ -241,7 +241,7 @@ func (h *Holder) applySchema(schema *internal.Schema) error {
 		// Create fields that don't exist.
 		for _, f := range index.Fields {
 			opt := decodeFieldOptions(f.Meta)
-			field, err := idx.CreateFieldIfNotExists(f.Name, *opt)
+			field, err := idx.createFieldIfNotExists(f.Name, *opt)
 			if err != nil {
 				return errors.Wrap(err, "creating field")
 			}
diff --git a/holder_internal_test.go b/holder_internal_test.go
index 425873005..75a3508fd 100644
--- a/holder_internal_test.go
+++ b/holder_internal_test.go
@@ -60,7 +60,7 @@ func newHolder() *tHolder {
 
 // MustCreateFieldIfNotExists returns a given field. Panic on error.
 func (h *tHolder) MustCreateFieldIfNotExists(index, field string) *Field {
-	f, err := h.MustCreateIndexIfNotExists(index, IndexOptions{}).CreateFieldIfNotExists(field, FieldOptions{})
+	f, err := h.MustCreateIndexIfNotExists(index, IndexOptions{}).CreateFieldIfNotExists(field, OptFieldTypeDefault())
 	if err != nil {
 		panic(err)
 	}
@@ -105,7 +105,7 @@ func TestHolder_Optn(t *testing.T) {
 
 		if idx, err := h.CreateIndex("foo", IndexOptions{}); err != nil {
 			t.Fatal(err)
-		} else if field, err := idx.CreateField("bar", FieldOptions{}); err != nil {
+		} else if field, err := idx.CreateField("bar", OptFieldTypeDefault()); err != nil {
 			t.Fatal(err)
 		} else if _, err := field.createViewIfNotExists(viewStandard); err != nil {
 			t.Fatal(err)
@@ -129,7 +129,7 @@ func TestHolder_Optn(t *testing.T) {
 
 		if idx, err := h.CreateIndex("foo", IndexOptions{}); err != nil {
 			t.Fatal(err)
-		} else if field, err := idx.CreateField("bar", FieldOptions{}); err != nil {
+		} else if field, err := idx.CreateField("bar", OptFieldTypeDefault()); err != nil {
 			t.Fatal(err)
 		} else if _, err := field.createViewIfNotExists(viewStandard); err != nil {
 			t.Fatal(err)
@@ -154,7 +154,7 @@ func TestHolder_Optn(t *testing.T) {
 
 		if idx, err := h.CreateIndex("foo", IndexOptions{}); err != nil {
 			t.Fatal(err)
-		} else if field, err := idx.CreateField("bar", FieldOptions{}); err != nil {
+		} else if field, err := idx.CreateField("bar", OptFieldTypeDefault()); err != nil {
 			t.Fatal(err)
 		} else if view, err := field.createViewIfNotExists(viewStandard); err != nil {
 			t.Fatal(err)
diff --git a/holder_test.go b/holder_test.go
index 078ab55f3..d4886ff30 100644
--- a/holder_test.go
+++ b/holder_test.go
@@ -98,7 +98,7 @@ func TestHolder_Open(t *testing.T) {
 
 		if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
 			t.Fatal(err)
-		} else if _, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil {
+		} else if _, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
 			t.Fatal(err)
 		} else if err := h.Holder.Close(); err != nil {
 			t.Fatal(err)
@@ -117,7 +117,7 @@ func TestHolder_Open(t *testing.T) {
 
 		if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
 			t.Fatal(err)
-		} else if _, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil {
+		} else if _, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
 			t.Fatal(err)
 		} else if err := h.Holder.Close(); err != nil {
 			t.Fatal(err)
@@ -135,7 +135,7 @@ func TestHolder_Open(t *testing.T) {
 
 		if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
 			t.Fatal(err)
-		} else if _, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil {
+		} else if _, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
 			t.Fatal(err)
 		} else if err := h.Holder.Close(); err != nil {
 			t.Fatal(err)
@@ -157,7 +157,7 @@ func TestHolder_Open(t *testing.T) {
 
 		if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
 			t.Fatal(err)
-		} else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil {
+		} else if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
 			t.Fatal(err)
 		} else if _, err := field.SetBit(0, 0, nil); err != nil {
 			t.Fatal(err)
@@ -178,7 +178,7 @@ func TestHolder_Open(t *testing.T) {
 
 		if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
 			t.Fatal(err)
-		} else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil {
+		} else if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
 			t.Fatal(err)
 		} else if _, err := field.SetBit(0, 0, nil); err != nil {
 			t.Fatal(err)
diff --git a/http/client_test.go b/http/client_test.go
index fb1105a27..00a9735a6 100644
--- a/http/client_test.go
+++ b/http/client_test.go
@@ -218,15 +218,10 @@ func TestClient_ImportValue(t *testing.T) {
 	hldr := test.Holder{Holder: holder}
 
 	fldName := "f"
-	fo := pilosa.FieldOptions{
-		Type: pilosa.FieldTypeInt,
-		Min:  -100,
-		Max:  100,
-	}
 
 	// Load bitmap into cache to ensure cache gets updated.
 	index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
-	field, err := index.CreateFieldIfNotExists(fldName, fo)
+	field, err := index.CreateFieldIfNotExists(fldName, pilosa.OptFieldTypeInt(-100, 100))
 	if err != nil {
 		t.Fatal(err)
 	}
diff --git a/index.go b/index.go
index 98f50eced..882bc75d3 100644
--- a/index.go
+++ b/index.go
@@ -287,7 +287,7 @@ func (i *Index) RecalculateCaches() {
 }
 
 // CreateField creates a field.
-func (i *Index) CreateField(name string, opt FieldOptions) (*Field, error) {
+func (i *Index) CreateField(name string, opts ...FieldOption) (*Field, error) {
 	i.mu.Lock()
 	defer i.mu.Unlock()
 
@@ -295,11 +295,40 @@ func (i *Index) CreateField(name string, opt FieldOptions) (*Field, error) {
 	if i.fields[name] != nil {
 		return nil, NewConflictError(ErrFieldExists)
 	}
-	return i.createField(name, opt)
+
+	// Apply functional options.
+	fo := FieldOptions{}
+	for _, opt := range opts {
+		err := opt(&fo)
+		if err != nil {
+			return nil, errors.Wrap(err, "applying option")
+		}
+	}
+
+	return i.createField(name, fo)
 }
 
 // CreateFieldIfNotExists creates a field with the given options if it doesn't exist.
-func (i *Index) CreateFieldIfNotExists(name string, opt FieldOptions) (*Field, error) {
+func (i *Index) CreateFieldIfNotExists(name string, opts FieldOption) (*Field, error) {
+	i.mu.Lock()
+	defer i.mu.Unlock()
+
+	// Find field in cache first.
+	if f := i.fields[name]; f != nil {
+		return f, nil
+	}
+
+	// Apply functional option.
+	fo := FieldOptions{}
+	err := opts(&fo)
+	if err != nil {
+		return nil, errors.Wrap(err, "applying option")
+	}
+
+	return i.createField(name, fo)
+}
+
+func (i *Index) createFieldIfNotExists(name string, opt FieldOptions) (*Field, error) {
 	i.mu.Lock()
 	defer i.mu.Unlock()
 
@@ -347,7 +376,7 @@ func (i *Index) createField(name string, opt FieldOptions) (*Field, error) {
 }
 
 func (i *Index) newField(path, name string) (*Field, error) {
-	f, err := NewField(path, i.name, name, FieldOptions{}) // TODO: NewField should be un-exported along with FieldOptions
+	f, err := NewField(path, i.name, name, OptFieldTypeDefault()) // TODO: NewField should be un-exported along with FieldOptions
 	if err != nil {
 		return nil, err
 	}
diff --git a/index_test.go b/index_test.go
index 5490e2978..bc412646a 100644
--- a/index_test.go
+++ b/index_test.go
@@ -33,7 +33,7 @@ func TestIndex_CreateFieldIfNotExists(t *testing.T) {
 	defer index.Close()
 
 	// Create field.
-	f, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{})
+	f, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeDefault())
 	if err != nil {
 		t.Fatal(err)
 	} else if f == nil {
@@ -41,7 +41,7 @@ func TestIndex_CreateFieldIfNotExists(t *testing.T) {
 	}
 
 	// Retrieve existing field.
-	other, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{})
+	other, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeDefault())
 	if err != nil {
 		t.Fatal(err)
 	} else if f.Field != other.Field {
@@ -61,10 +61,7 @@ func TestIndex_CreateField(t *testing.T) {
 			defer index.Close()
 
 			// Create field with explicit quantum.
-			f, err := index.CreateField("f", pilosa.FieldOptions{
-				Type:        pilosa.FieldTypeTime,
-				TimeQuantum: pilosa.TimeQuantum("YMDH"),
-			})
+			f, err := index.CreateField("f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")))
 			if err != nil {
 				t.Fatal(err)
 			} else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
@@ -80,11 +77,7 @@ func TestIndex_CreateField(t *testing.T) {
 			defer index.Close()
 
 			// Create field with schema and verify it exists.
-			if f, err := index.CreateField("f", pilosa.FieldOptions{
-				Type: pilosa.FieldTypeInt,
-				Min:  10,
-				Max:  20,
-			}); err != nil {
+			if f, err := index.CreateField("f", pilosa.OptFieldTypeInt(10, 20)); err != nil {
 				t.Fatal(err)
 			} else if !reflect.DeepEqual(f.Type(), pilosa.FieldTypeInt) {
 				t.Fatalf("unexpected type: %#v", f.Type())
@@ -184,7 +177,7 @@ func TestIndex_DeleteField(t *testing.T) {
 	defer index.Close()
 
 	// Create field.
-	if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{}); err != nil {
+	if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeDefault()); err != nil {
 		t.Fatal(err)
 	}
 
diff --git a/server.go b/server.go
index 56195b427..ba058d134 100644
--- a/server.go
+++ b/server.go
@@ -455,7 +455,7 @@ func (s *Server) receiveMessage(pb proto.Message) error {
 			return fmt.Errorf("Local Index not found: %s", obj.Index)
 		}
 		opt := decodeFieldOptions(obj.Meta)
-		_, err := idx.CreateField(obj.Field, *opt)
+		_, err := idx.createField(obj.Field, *opt)
 		if err != nil {
 			return err
 		}
diff --git a/server/handler_test.go b/server/handler_test.go
index e49b1eb0f..a5ba8c398 100644
--- a/server/handler_test.go
+++ b/server/handler_test.go
@@ -62,17 +62,17 @@ func TestHandler_Endpoints(t *testing.T) {
 
 	i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
 	i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{})
-	if f, err := i0.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil {
+	if f, err := i0.CreateFieldIfNotExists("f1", pilosa.OptFieldTypeDefault()); err != nil {
 		t.Fatal(err)
 	} else if _, err := f.SetBit(0, 0, nil); err != nil {
 		t.Fatal(err)
 	}
-	if f, err := i1.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil {
+	if f, err := i1.CreateFieldIfNotExists("f0", pilosa.OptFieldTypeDefault()); err != nil {
 		t.Fatal(err)
 	} else if _, err := f.SetBit(0, 0, nil); err != nil {
 		t.Fatal(err)
 	}
-	if _, err := i0.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil {
+	if _, err := i0.CreateFieldIfNotExists("f0", pilosa.OptFieldTypeDefault()); err != nil {
 		t.Fatal(err)
 	}
 
@@ -401,7 +401,7 @@ func TestHandler_Endpoints(t *testing.T) {
 
 	t.Run("Field delete", func(t *testing.T) {
 		i := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
-		if _, err := i.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil {
+		if _, err := i.CreateFieldIfNotExists("f1", pilosa.OptFieldTypeDefault()); err != nil {
 			t.Fatal(err)
 		}
 		w := httptest.NewRecorder()
@@ -453,7 +453,7 @@ func TestHandler_Endpoints(t *testing.T) {
 		}
 	})
 
-	meta, err := i.CreateFieldIfNotExists("meta", pilosa.FieldOptions{})
+	meta, err := i.CreateFieldIfNotExists("meta", pilosa.OptFieldTypeDefault())
 	if err != nil {
 		t.Fatal(err)
 	}
diff --git a/test/field.go b/test/field.go
index 345deadf8..7a0439026 100644
--- a/test/field.go
+++ b/test/field.go
@@ -28,12 +28,12 @@ type Field struct {
 }
 
 // NewField returns a new instance of Field d/0.
-func NewField(options pilosa.FieldOptions) *Field {
+func NewField(opts pilosa.FieldOption) *Field {
 	path, err := ioutil.TempDir("", "pilosa-field-")
 	if err != nil {
 		panic(err)
 	}
-	field, err := pilosa.NewField(path, "i", "f", options)
+	field, err := pilosa.NewField(path, "i", "f", opts)
 	if err != nil {
 		panic(err)
 	}
@@ -41,8 +41,8 @@ func NewField(options pilosa.FieldOptions) *Field {
 }
 
 // MustOpenField returns a new, opened field at a temporary path. Panic on error.
-func MustOpenField(options pilosa.FieldOptions) *Field {
-	f := NewField(options)
+func MustOpenField(opts pilosa.FieldOption) *Field {
+	f := NewField(opts)
 	if err := f.Open(); err != nil {
 		panic(err)
 	}
@@ -63,7 +63,7 @@ func (f *Field) Reopen() error {
 	}
 
 	path, index, name := f.Path(), f.Index(), f.Name()
-	f.Field, err = pilosa.NewField(path, index, name, pilosa.FieldOptions{})
+	f.Field, err = pilosa.NewField(path, index, name, pilosa.OptFieldTypeDefault())
 	if err != nil {
 		return err
 	}
@@ -76,7 +76,7 @@ func (f *Field) Reopen() error {
 
 // Ensure field can set its cache
 func TestField_SetCacheSize(t *testing.T) {
-	f := MustOpenField(pilosa.FieldOptions{})
+	f := MustOpenField(pilosa.OptFieldTypeDefault())
 	defer f.Close()
 	cacheSize := uint32(100)
 
diff --git a/test/holder.go b/test/holder.go
index 94d1c778f..9cc96fe14 100644
--- a/test/holder.go
+++ b/test/holder.go
@@ -83,7 +83,7 @@ func (h *Holder) MustCreateIndexIfNotExists(index string, opt pilosa.IndexOption
 
 // MustCreateFieldIfNotExists returns a given field. Panic on error.
 func (h *Holder) MustCreateFieldIfNotExists(index, field string) *Field {
-	f, err := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}).CreateFieldIfNotExists(field, pilosa.FieldOptions{})
+	f, err := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}).CreateFieldIfNotExists(field, pilosa.OptFieldTypeDefault())
 	if err != nil {
 		panic(err)
 	}
@@ -93,7 +93,7 @@ func (h *Holder) MustCreateFieldIfNotExists(index, field string) *Field {
 // Row returns a Row for a given field.
 func (h *Holder) Row(index, field string, rowID uint64) *pilosa.Row {
 	idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
-	f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{})
+	f, err := idx.CreateFieldIfNotExists(field, pilosa.OptFieldTypeDefault())
 	if err != nil {
 		panic(err)
 	}
@@ -106,7 +106,7 @@ func (h *Holder) Row(index, field string, rowID uint64) *pilosa.Row {
 
 func (h *Holder) RowAttrStore(index, field string) pilosa.AttrStore {
 	idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
-	f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{})
+	f, err := idx.CreateFieldIfNotExists(field, pilosa.OptFieldTypeDefault())
 	if err != nil {
 		panic(err)
 	}
@@ -115,7 +115,7 @@ func (h *Holder) RowAttrStore(index, field string) pilosa.AttrStore {
 
 func (h *Holder) RowTime(index, field string, rowID uint64, t time.Time, quantum string) *pilosa.Row {
 	idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
-	f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{})
+	f, err := idx.CreateFieldIfNotExists(field, pilosa.OptFieldTypeDefault())
 	if err != nil {
 		panic(err)
 	}
@@ -129,7 +129,7 @@ func (h *Holder) RowTime(index, field string, rowID uint64, t time.Time, quantum
 // SetBit clears a bit on the given field.
 func (h *Holder) SetBit(index, field string, rowID, columnID uint64) {
 	idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
-	f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{})
+	f, err := idx.CreateFieldIfNotExists(field, pilosa.OptFieldTypeDefault())
 	if err != nil {
 		panic(err)
 	}
@@ -142,7 +142,7 @@ func (h *Holder) SetBit(index, field string, rowID, columnID uint64) {
 // ClearBit clears a bit on the given field.
 func (h *Holder) ClearBit(index, field string, rowID, columnID uint64) {
 	idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
-	f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{})
+	f, err := idx.CreateFieldIfNotExists(field, pilosa.OptFieldTypeDefault())
 	if err != nil {
 		panic(err)
 	}
diff --git a/test/index.go b/test/index.go
index 69c18682c..9b7c1684b 100644
--- a/test/index.go
+++ b/test/index.go
@@ -74,8 +74,8 @@ func (i *Index) Reopen() error {
 }
 
 // CreateField creates a field with the given options.
-func (i *Index) CreateField(name string, opt pilosa.FieldOptions) (*Field, error) {
-	f, err := i.Index.CreateField(name, opt)
+func (i *Index) CreateField(name string, opts ...pilosa.FieldOption) (*Field, error) {
+	f, err := i.Index.CreateField(name, opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -83,8 +83,8 @@ func (i *Index) CreateField(name string, opt pilosa.FieldOptions) (*Field, error
 }
 
 // CreateFieldIfNotExists creates a field with the given options if it doesn't exist.
-func (i *Index) CreateFieldIfNotExists(name string, opt pilosa.FieldOptions) (*Field, error) {
-	f, err := i.Index.CreateFieldIfNotExists(name, opt)
+func (i *Index) CreateFieldIfNotExists(name string, opts pilosa.FieldOption) (*Field, error) {
+	f, err := i.Index.CreateFieldIfNotExists(name, opts)
 	if err != nil {
 		return nil, err
 	}
diff --git a/utils_internal_test.go b/utils_internal_test.go
index ca7bd1fa7..bbcf300cd 100644
--- a/utils_internal_test.go
+++ b/utils_internal_test.go
@@ -104,13 +104,13 @@ func (t *ClusterCluster) CreateIndex(name string) error {
 	return nil
 }
 
-func (t *ClusterCluster) CreateField(index, field string, opt FieldOptions) error {
+func (t *ClusterCluster) CreateField(index, field string, opts FieldOption) error {
 	for _, c := range t.Clusters {
 		idx, err := c.holder.CreateIndexIfNotExists(index, IndexOptions{})
 		if err != nil {
 			return err
 		}
-		if _, err := idx.CreateField(field, opt); err != nil {
+		if _, err := idx.CreateField(field, opts); err != nil {
 			return err
 		}
 	}

From 3e288aa3910e7f3b50bd67a6536481bf7de4b31a Mon Sep 17 00:00:00 2001
From: Travis Turner <travis@pilosa.com>
Date: Wed, 4 Jul 2018 21:39:01 -0500
Subject: [PATCH 10/32] un-export field.FieldOptions

---
 api.go          |  2 +-
 field.go        | 46 +++++++++++++++++++++++-----------------------
 http/handler.go |  2 +-
 index.go        | 10 +++++-----
 4 files changed, 30 insertions(+), 30 deletions(-)

diff --git a/api.go b/api.go
index f0e2dbfb0..f9efbdb00 100644
--- a/api.go
+++ b/api.go
@@ -244,7 +244,7 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str
 	}
 
 	// Apply functional option.
-	fo := FieldOptions{}
+	fo := fieldOptions{}
 	err := opts(&fo)
 	if err != nil {
 		return nil, errors.Wrap(err, "applying option")
diff --git a/field.go b/field.go
index 2b78176e1..f2cb711c2 100644
--- a/field.go
+++ b/field.go
@@ -68,25 +68,25 @@ type Field struct {
 	Stats       StatsClient
 
 	// Field options.
-	options FieldOptions
+	options fieldOptions
 
 	bsiGroups []*bsiGroup
 
 	Logger Logger
 }
 
-// FieldOption is a functional option type for pilosa.FieldOptions.
-type FieldOption func(fo *FieldOptions) error
+// FieldOption is a functional option type for pilosa.fieldOptions.
+type FieldOption func(fo *fieldOptions) error
 
 func OptFieldKeys() FieldOption {
-	return func(fo *FieldOptions) error {
+	return func(fo *fieldOptions) error {
 		fo.Keys = true
 		return nil
 	}
 }
 
 func OptFieldTypeDefault() FieldOption {
-	return func(fo *FieldOptions) error {
+	return func(fo *fieldOptions) error {
 		if fo.Type != "" {
 			return errors.Errorf("field type is already set to: %s", fo.Type)
 		}
@@ -98,7 +98,7 @@ func OptFieldTypeDefault() FieldOption {
 }
 
 func OptFieldTypeSet(cacheType string, cacheSize uint32) FieldOption {
-	return func(fo *FieldOptions) error {
+	return func(fo *fieldOptions) error {
 		if fo.Type != "" {
 			return errors.Errorf("field type is already set to: %s", fo.Type)
 		}
@@ -110,7 +110,7 @@ func OptFieldTypeSet(cacheType string, cacheSize uint32) FieldOption {
 }
 
 func OptFieldTypeInt(min, max int64) FieldOption {
-	return func(fo *FieldOptions) error {
+	return func(fo *fieldOptions) error {
 		if fo.Type != "" {
 			return errors.Errorf("field type is already set to: %s", fo.Type)
 		}
@@ -125,7 +125,7 @@ func OptFieldTypeInt(min, max int64) FieldOption {
 }
 
 func OptFieldTypeTime(timeQuantum TimeQuantum) FieldOption {
-	return func(fo *FieldOptions) error {
+	return func(fo *fieldOptions) error {
 		if fo.Type != "" {
 			return errors.Errorf("field type is already set to: %s", fo.Type)
 		}
@@ -146,7 +146,7 @@ func NewField(path, index, name string, opts FieldOption) (*Field, error) {
 	}
 
 	// Apply functional option.
-	fo := FieldOptions{}
+	fo := fieldOptions{}
 	err = opts(&fo)
 	if err != nil {
 		return nil, errors.Wrap(err, "applying option")
@@ -240,7 +240,7 @@ func (f *Field) CacheSize() uint32 {
 }
 
 // Options returns all options for this field.
-func (f *Field) Options() FieldOptions {
+func (f *Field) Options() fieldOptions {
 	f.mu.RLock()
 	defer f.mu.RUnlock()
 	return f.options
@@ -358,7 +358,7 @@ func (f *Field) saveMeta() error {
 }
 
 // applyOptions configures the field based on opt.
-func (f *Field) applyOptions(opt FieldOptions) error {
+func (f *Field) applyOptions(opt fieldOptions) error {
 	switch opt.Type {
 	case FieldTypeSet, "":
 		f.options.Type = FieldTypeSet
@@ -1102,7 +1102,7 @@ func (f *Field) ImportValue(columnIDs []uint64, values []int64) error {
 func (f *Field) MarshalJSON() ([]byte, error) {
 	thing := struct {
 		Name    string
-		Options FieldOptions
+		Options fieldOptions
 		Views   []*viewInfo
 	}{
 		Name:    f.Name(),
@@ -1142,7 +1142,7 @@ func (p fieldSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() }
 // FieldInfo represents schema information for a field.
 type FieldInfo struct {
 	Name    string       `json:"name"`
-	Options FieldOptions `json:"options"`
+	Options fieldOptions `json:"options"`
 	Views   []*viewInfo  `json:"views,omitempty"`
 }
 
@@ -1152,8 +1152,8 @@ func (p fieldInfoSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
 func (p fieldInfoSlice) Len() int           { return len(p) }
 func (p fieldInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
 
-// FieldOptions represents options to set when initializing a field.
-type FieldOptions struct {
+// fieldOptions represents options to set when initializing a field.
+type fieldOptions struct {
 	Type        string      `json:"type,omitempty"`
 	CacheType   string      `json:"cacheType,omitempty"`
 	CacheSize   uint32      `json:"cacheSize,omitempty"`
@@ -1163,11 +1163,11 @@ type FieldOptions struct {
 	Keys        bool        `json:"keys,omitempty"`
 }
 
-// applyDefaultOptions returns a new FieldOptions object
+// applyDefaultOptions returns a new fieldOptions object
 // with default values if o does not contain a valid type.
-func applyDefaultOptions(o FieldOptions) FieldOptions {
+func applyDefaultOptions(o fieldOptions) fieldOptions {
 	if o.Type == "" {
-		return FieldOptions{
+		return fieldOptions{
 			Type:      DefaultFieldType,
 			CacheType: DefaultCacheType,
 			CacheSize: DefaultCacheSize,
@@ -1177,11 +1177,11 @@ func applyDefaultOptions(o FieldOptions) FieldOptions {
 }
 
 // Encode converts o into its internal representation.
-func (o *FieldOptions) Encode() *internal.FieldOptions {
+func (o *fieldOptions) Encode() *internal.FieldOptions {
 	return encodeFieldOptions(o)
 }
 
-func encodeFieldOptions(o *FieldOptions) *internal.FieldOptions {
+func encodeFieldOptions(o *fieldOptions) *internal.FieldOptions {
 	if o == nil {
 		return nil
 	}
@@ -1196,11 +1196,11 @@ func encodeFieldOptions(o *FieldOptions) *internal.FieldOptions {
 	}
 }
 
-func decodeFieldOptions(options *internal.FieldOptions) *FieldOptions {
+func decodeFieldOptions(options *internal.FieldOptions) *fieldOptions {
 	if options == nil {
 		return nil
 	}
-	return &FieldOptions{
+	return &fieldOptions{
 		Type:        options.Type,
 		CacheType:   options.CacheType,
 		CacheSize:   options.CacheSize,
@@ -1211,7 +1211,7 @@ func decodeFieldOptions(options *internal.FieldOptions) *FieldOptions {
 	}
 }
 
-func (o *FieldOptions) MarshalJSON() ([]byte, error) {
+func (o *fieldOptions) MarshalJSON() ([]byte, error) {
 	switch o.Type {
 	case FieldTypeSet:
 		return json.Marshal(struct {
diff --git a/http/handler.go b/http/handler.go
index 9195f05e9..1089c5329 100644
--- a/http/handler.go
+++ b/http/handler.go
@@ -671,7 +671,7 @@ type postFieldRequest struct {
 	Options fieldOptions `json:"options"`
 }
 
-// fieldOptions tracks pilosa.FieldOptions. It is made up of pointers to values,
+// fieldOptions tracks pilosa.fieldOptions. It is made up of pointers to values,
 // and used for input validation.
 type fieldOptions struct {
 	Type        string              `json:"type,omitempty"`
diff --git a/index.go b/index.go
index 882bc75d3..bf0e55290 100644
--- a/index.go
+++ b/index.go
@@ -297,7 +297,7 @@ func (i *Index) CreateField(name string, opts ...FieldOption) (*Field, error) {
 	}
 
 	// Apply functional options.
-	fo := FieldOptions{}
+	fo := fieldOptions{}
 	for _, opt := range opts {
 		err := opt(&fo)
 		if err != nil {
@@ -319,7 +319,7 @@ func (i *Index) CreateFieldIfNotExists(name string, opts FieldOption) (*Field, e
 	}
 
 	// Apply functional option.
-	fo := FieldOptions{}
+	fo := fieldOptions{}
 	err := opts(&fo)
 	if err != nil {
 		return nil, errors.Wrap(err, "applying option")
@@ -328,7 +328,7 @@ func (i *Index) CreateFieldIfNotExists(name string, opts FieldOption) (*Field, e
 	return i.createField(name, fo)
 }
 
-func (i *Index) createFieldIfNotExists(name string, opt FieldOptions) (*Field, error) {
+func (i *Index) createFieldIfNotExists(name string, opt fieldOptions) (*Field, error) {
 	i.mu.Lock()
 	defer i.mu.Unlock()
 
@@ -340,7 +340,7 @@ func (i *Index) createFieldIfNotExists(name string, opt FieldOptions) (*Field, e
 	return i.createField(name, opt)
 }
 
-func (i *Index) createField(name string, opt FieldOptions) (*Field, error) {
+func (i *Index) createField(name string, opt fieldOptions) (*Field, error) {
 	if name == "" {
 		return nil, errors.New("field name required")
 	} else if opt.CacheType != "" && !isValidCacheType(opt.CacheType) {
@@ -376,7 +376,7 @@ func (i *Index) createField(name string, opt FieldOptions) (*Field, error) {
 }
 
 func (i *Index) newField(path, name string) (*Field, error) {
-	f, err := NewField(path, i.name, name, OptFieldTypeDefault()) // TODO: NewField should be un-exported along with FieldOptions
+	f, err := NewField(path, i.name, name, OptFieldTypeDefault())
 	if err != nil {
 		return nil, err
 	}

From cd8c63c125935bba86a0b658eac13de50a4ffbc1 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Wed, 4 Jul 2018 21:43:18 -0500
Subject: [PATCH 11/32] tests passing

---
 api.go                   | 17 ++++++++---------
 cluster_internal_test.go | 41 ++++++++++++++++++++--------------------
 field.go                 |  2 +-
 view.go                  |  3 +--
 4 files changed, 30 insertions(+), 33 deletions(-)

diff --git a/api.go b/api.go
index 65ff90870..73bdd6665 100644
--- a/api.go
+++ b/api.go
@@ -185,12 +185,11 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index
 	}
 	// Send the create index message to all nodes.
 	err = api.server.SendSync(
-		&internal.CreateIndexMessage{
+		&CreateIndexMessage{
 			Index: indexName,
-			Meta:  options.Encode(),
+			Meta:  &options,
 		})
 	if err != nil {
-		api.server.logger.Printf("problem sending CreateIndex message: %s", err)
 		return nil, errors.Wrap(err, "sending CreateIndex message")
 	}
 	api.holder.Stats.Count("createIndex", 1, 1.0)
@@ -224,7 +223,7 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error {
 	}
 	// Send the delete index message to all nodes.
 	err = api.server.SendSync(
-		&internal.DeleteIndexMessage{
+		&DeleteIndexMessage{
 			Index: indexName,
 		})
 	if err != nil {
@@ -264,10 +263,10 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str
 
 	// Send the create field message to all nodes.
 	err = api.server.SendSync(
-		&internal.CreateFieldMessage{
+		&CreateFieldMessage{
 			Index: indexName,
 			Field: fieldName,
-			Meta:  fo.Encode(),
+			Meta:  &fo,
 		})
 	if err != nil {
 		api.server.logger.Printf("problem sending CreateField message: %s", err)
@@ -311,7 +310,7 @@ func (api *API) DeleteField(ctx context.Context, indexName string, fieldName str
 
 	// Send the delete field message to all nodes.
 	err := api.server.SendSync(
-		&internal.DeleteFieldMessage{
+		&DeleteFieldMessage{
 			Index: indexName,
 			Field: fieldName,
 		})
@@ -489,7 +488,7 @@ func (api *API) RecalculateCaches(ctx context.Context) error {
 		return errors.Wrap(err, "validating api method")
 	}
 
-	err := api.server.SendSync(&internal.RecalculateCaches{})
+	err := api.server.SendSync(&RecalculateCaches{})
 	if err != nil {
 		return errors.Wrap(err, "broacasting message")
 	}
@@ -568,7 +567,7 @@ func (api *API) DeleteView(ctx context.Context, indexName string, fieldName stri
 
 	// Send the delete view message to all nodes.
 	err := api.server.SendSync(
-		&internal.DeleteViewMessage{
+		&DeleteViewMessage{
 			Index: indexName,
 			Field: fieldName,
 			View:  viewName,
diff --git a/cluster_internal_test.go b/cluster_internal_test.go
index aee4ea09b..e209a8f80 100644
--- a/cluster_internal_test.go
+++ b/cluster_internal_test.go
@@ -24,7 +24,6 @@ import (
 	"testing/quick"
 
 	"github.com/davecgh/go-spew/spew"
-	"github.com/pilosa/pilosa/internal"
 	"github.com/pkg/errors"
 )
 
@@ -175,19 +174,19 @@ func TestFragSources(t *testing.T) {
 		from     *cluster
 		to       *cluster
 		idx      *Index
-		expected map[string][]*internal.ResizeSource
+		expected map[string][]*ResizeSource
 		err      string
 	}{
 		{
 			from: c1,
 			to:   c2,
 			idx:  idx,
-			expected: map[string][]*internal.ResizeSource{
-				"node0": []*internal.ResizeSource{},
-				"node1": []*internal.ResizeSource{},
-				"node2": []*internal.ResizeSource{
-					{&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)},
-					{&internal.Node{"node1", &internal.URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(2)},
+			expected: map[string][]*ResizeSource{
+				"node0": []*ResizeSource{},
+				"node1": []*ResizeSource{},
+				"node2": []*ResizeSource{
+					{&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)},
+					{&Node{"node1", URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(2)},
 				},
 			},
 			err: "",
@@ -196,13 +195,13 @@ func TestFragSources(t *testing.T) {
 			from: c4,
 			to:   c3,
 			idx:  idx,
-			expected: map[string][]*internal.ResizeSource{
-				"node0": []*internal.ResizeSource{
-					{&internal.Node{"node1", &internal.URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(1)},
+			expected: map[string][]*ResizeSource{
+				"node0": []*ResizeSource{
+					{&Node{"node1", URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(1)},
 				},
-				"node1": []*internal.ResizeSource{
-					{&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)},
-					{&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(2)},
+				"node1": []*ResizeSource{
+					{&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)},
+					{&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(2)},
 				},
 			},
 			err: "",
@@ -211,15 +210,15 @@ func TestFragSources(t *testing.T) {
 			from: c5,
 			to:   c4,
 			idx:  idx,
-			expected: map[string][]*internal.ResizeSource{
-				"node0": []*internal.ResizeSource{
-					{&internal.Node{"node2", &internal.URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(0)},
-					{&internal.Node{"node2", &internal.URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(2)},
+			expected: map[string][]*ResizeSource{
+				"node0": []*ResizeSource{
+					{&Node{"node2", URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(0)},
+					{&Node{"node2", URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(2)},
 				},
-				"node1": []*internal.ResizeSource{
-					{&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(3)},
+				"node1": []*ResizeSource{
+					{&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(3)},
 				},
-				"node2": []*internal.ResizeSource{},
+				"node2": []*ResizeSource{},
 			},
 			err: "",
 		},
diff --git a/field.go b/field.go
index 76be9cf3f..eea6bb10e 100644
--- a/field.go
+++ b/field.go
@@ -605,7 +605,7 @@ func (f *Field) createViewIfNotExists(name string) (*view, error) {
 	if created {
 		// Broadcast view creation to the cluster.
 		err = f.broadcaster.SendSync(
-			&internal.CreateViewMessage{
+			&CreateViewMessage{
 				Index: f.index,
 				Field: f.name,
 				View:  name,
diff --git a/view.go b/view.go
index fd5306b85..0f2e189cd 100644
--- a/view.go
+++ b/view.go
@@ -22,7 +22,6 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/pilosa/pilosa/internal"
 	"github.com/pilosa/pilosa/pql"
 	"github.com/pkg/errors"
 )
@@ -232,7 +231,7 @@ func (v *view) createFragmentIfNotExists(shard uint64) (*fragment, error) {
 
 		// Send the create shard message to all nodes.
 		err := v.broadcaster.SendSync(
-			&internal.CreateShardMessage{
+			&CreateShardMessage{
 				Index: v.index,
 				Shard: shard,
 			})

From bae5ee36efecc591669a1a2f1331c86306b976ca Mon Sep 17 00:00:00 2001
From: Travis Turner <travis@pilosa.com>
Date: Wed, 4 Jul 2018 22:24:05 -0500
Subject: [PATCH 12/32] add support for OptFieldKeys() to http field creation

---
 api.go          | 14 ++++++++------
 http/handler.go | 15 ++++++++++-----
 2 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/api.go b/api.go
index f9efbdb00..8ca101c61 100644
--- a/api.go
+++ b/api.go
@@ -238,16 +238,18 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error {
 // CreateField makes the named field in the named index with the given options.
 // This method currently only takes a single functional option, but that may be
 // changed in the future to support multiple options.
-func (api *API) CreateField(ctx context.Context, indexName string, fieldName string, opts FieldOption) (*Field, error) {
+func (api *API) CreateField(ctx context.Context, indexName string, fieldName string, opts ...FieldOption) (*Field, error) {
 	if err := api.validate(apiCreateField); err != nil {
 		return nil, errors.Wrap(err, "validating api method")
 	}
 
-	// Apply functional option.
+	// Apply functional options.
 	fo := fieldOptions{}
-	err := opts(&fo)
-	if err != nil {
-		return nil, errors.Wrap(err, "applying option")
+	for _, opt := range opts {
+		err := opt(&fo)
+		if err != nil {
+			return nil, errors.Wrap(err, "applying option")
+		}
 	}
 
 	// Find index.
@@ -257,7 +259,7 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str
 	}
 
 	// Create field.
-	field, err := index.CreateField(fieldName, opts)
+	field, err := index.CreateField(fieldName, opts...)
 	if err != nil {
 		return nil, errors.Wrap(err, "creating field")
 	}
diff --git a/http/handler.go b/http/handler.go
index 1089c5329..8b2015094 100644
--- a/http/handler.go
+++ b/http/handler.go
@@ -653,17 +653,22 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Convert json options into functional options.
-	var fos pilosa.FieldOption
+	var fos []pilosa.FieldOption
 	switch req.Options.Type {
 	case pilosa.FieldTypeSet:
-		fos = pilosa.OptFieldTypeSet(*req.Options.CacheType, *req.Options.CacheSize)
+		fos = append(fos, pilosa.OptFieldTypeSet(*req.Options.CacheType, *req.Options.CacheSize))
 	case pilosa.FieldTypeInt:
-		fos = pilosa.OptFieldTypeInt(*req.Options.Min, *req.Options.Max)
+		fos = append(fos, pilosa.OptFieldTypeInt(*req.Options.Min, *req.Options.Max))
 	case pilosa.FieldTypeTime:
-		fos = pilosa.OptFieldTypeTime(*req.Options.TimeQuantum)
+		fos = append(fos, pilosa.OptFieldTypeTime(*req.Options.TimeQuantum))
+	}
+	if req.Options.Keys != nil {
+		if *req.Options.Keys {
+			fos = append(fos, pilosa.OptFieldKeys())
+		}
 	}
 
-	_, err = h.API.CreateField(r.Context(), indexName, fieldName, fos)
+	_, err = h.API.CreateField(r.Context(), indexName, fieldName, fos...)
 	resp.write(w, err)
 }
 

From 1b2aaa26bf13c1b42b902a09ff125eee7ca32080 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Thu, 5 Jul 2018 09:26:58 -0500
Subject: [PATCH 13/32] export NodeEvent

---
 broadcast.go           |  2 +-
 cluster.go             | 14 +++++++-------
 event.go               |  4 ++--
 server.go              |  2 +-
 utils_internal_test.go |  2 +-
 5 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/broadcast.go b/broadcast.go
index 0a33d4ca7..d3c4a2562 100644
--- a/broadcast.go
+++ b/broadcast.go
@@ -148,7 +148,7 @@ func encode(m Message) proto.Message {
 		return encodeNodeStateMessage(mt)
 	case *RecalculateCaches:
 		return encodeRecalculateCaches(mt)
-	case *nodeEvent:
+	case *NodeEvent:
 		return encodeNodeEventMessage(mt)
 	case *NodeStatus:
 		return encodeNodeStatus(mt)
diff --git a/cluster.go b/cluster.go
index e97ba46e4..2d25ebeaa 100644
--- a/cluster.go
+++ b/cluster.go
@@ -856,7 +856,7 @@ func (c *cluster) waitForStarted() error {
 		// TODO: Because the normal code path already sends a NodeJoin event (via
 		// memberlist), this it a bit redundant in most cases. Perhaps determine
 		// that the node has been restarted and don't do this step.
-		msg := &nodeEvent{
+		msg := &NodeEvent{
 			Event: NodeJoin,
 			Node:  c.Node,
 		}
@@ -1540,7 +1540,7 @@ func (c *cluster) considerTopology() error {
 }
 
 // ReceiveEvent represents an implementation of EventHandler.
-func (c *cluster) ReceiveEvent(e *nodeEvent) error {
+func (c *cluster) ReceiveEvent(e *NodeEvent) error {
 	// Ignore events sent from this node.
 	if e.Node.ID == c.Node.ID {
 		return nil
@@ -1966,8 +1966,8 @@ func DecodeNode(node *internal.Node) *Node {
 	}
 }
 
-func DecodeNodeEvent(ne *internal.NodeEventMessage) *nodeEvent {
-	return &nodeEvent{
+func DecodeNodeEvent(ne *internal.NodeEventMessage) *NodeEvent {
+	return &NodeEvent{
 		Event: NodeEventType(ne.Event),
 		Node:  DecodeNode(ne.Node),
 	}
@@ -2223,15 +2223,15 @@ func decodeNodeStateMessage(pb *internal.NodeStateMessage) *NodeStateMessage {
 	}
 }
 
-func encodeNodeEventMessage(m *nodeEvent) *internal.NodeEventMessage {
+func encodeNodeEventMessage(m *NodeEvent) *internal.NodeEventMessage {
 	return &internal.NodeEventMessage{
 		Event: uint32(m.Event),
 		Node:  EncodeNode(m.Node),
 	}
 }
 
-func decodeNodeEventMessage(pb *internal.NodeEventMessage) *nodeEvent {
-	return &nodeEvent{
+func decodeNodeEventMessage(pb *internal.NodeEventMessage) *NodeEvent {
+	return &NodeEvent{
 		Event: NodeEventType(pb.Event),
 		Node:  DecodeNode(pb.Node),
 	}
diff --git a/event.go b/event.go
index aa1e0e890..0d5e59e99 100644
--- a/event.go
+++ b/event.go
@@ -23,8 +23,8 @@ const (
 	NodeUpdate
 )
 
-// nodeEvent is a single event related to node activity in the cluster.
-type nodeEvent struct {
+// NodeEvent is a single event related to node activity in the cluster.
+type NodeEvent struct {
 	Event NodeEventType
 	Node  *Node
 }
diff --git a/server.go b/server.go
index 7199b7178..b57935acb 100644
--- a/server.go
+++ b/server.go
@@ -506,7 +506,7 @@ func (s *Server) receiveMessage(m Message) error {
 		}
 	case *RecalculateCaches:
 		s.holder.RecalculateCaches()
-	case *nodeEvent:
+	case *NodeEvent:
 		s.cluster.ReceiveEvent(obj)
 	case *NodeStatus:
 		s.handleRemoteStatus(obj)
diff --git a/utils_internal_test.go b/utils_internal_test.go
index f7c11d6a7..4b9f6870a 100644
--- a/utils_internal_test.go
+++ b/utils_internal_test.go
@@ -161,7 +161,7 @@ func (t *ClusterCluster) addNode() error {
 	// Send NodeJoin event to coordinator.
 	if id > 0 {
 		coord := t.Clusters[0]
-		ev := &nodeEvent{
+		ev := &NodeEvent{
 			Event: NodeJoin,
 			Node:  c.Node,
 		}

From 2cec75e39931f46d31842d16f273d67826fcc00c Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Thu, 5 Jul 2018 12:00:42 -0500
Subject: [PATCH 14/32] add proto encoding subpackage and use for send and
 receive message

---
 api.go                  |   9 +-
 broadcast.go            | 116 +++++---
 client.go               |   5 +-
 cluster.go              |   4 +-
 encoding/proto/proto.go | 594 ++++++++++++++++++++++++++++++++++++++++
 field.go                |   6 +-
 gossip/gossip.go        |   6 +-
 holder.go               |   2 +-
 http/client.go          |  14 +-
 server.go               |  26 +-
 server/server.go        |  14 +-
 uri.go                  |  74 ++---
 uri_internal_test.go    |  24 +-
 view.go                 |   6 +-
 14 files changed, 778 insertions(+), 122 deletions(-)
 create mode 100644 encoding/proto/proto.go

diff --git a/api.go b/api.go
index 73bdd6665..09f0531d5 100644
--- a/api.go
+++ b/api.go
@@ -509,14 +509,15 @@ func (api *API) ClusterMessage(ctx context.Context, reqBody io.Reader) error {
 		return errors.Wrap(err, "reading body")
 	}
 
-	// Marshal into request object.
-	pb, err := UnmarshalMessage(body)
+	typ := body[0]
+	msg := getMessage(typ)
+	err = api.server.serializer.Unmarshal(body[1:], msg)
 	if err != nil {
-		return errors.Wrap(err, "unmarshaling message")
+		return errors.Wrap(err, "deserializing cluster message")
 	}
 
 	// Forward the error message.
-	if err := api.server.receiveMessage(decode(pb)); err != nil {
+	if err := api.server.receiveMessage(msg); err != nil {
 		return errors.Wrap(err, "receiving message")
 	}
 	return nil
diff --git a/broadcast.go b/broadcast.go
index d3c4a2562..1da6ca57f 100644
--- a/broadcast.go
+++ b/broadcast.go
@@ -23,6 +23,12 @@ import (
 	"github.com/pkg/errors"
 )
 
+// Serializer is an interface for serializing pilosa types to bytes and back.
+type Serializer interface {
+	Marshal(Message) ([]byte, error)
+	Unmarshal([]byte, Message) error
+}
+
 // broadcaster is an interface for broadcasting messages.
 type broadcaster interface {
 	SendSync(Message) error
@@ -118,42 +124,82 @@ func MarshalMessage(m proto.Message) ([]byte, error) {
 	return append([]byte{typ}, buf...), nil
 }
 
-func encode(m Message) proto.Message {
-	switch mt := m.(type) {
-	case *CreateShardMessage:
-		return encodeCreateShardMessage(mt)
-	case *CreateIndexMessage:
-		return encodeCreateIndexMessage(mt)
-	case *DeleteIndexMessage:
-		return encodeDeleteIndexMessage(mt)
-	case *CreateFieldMessage:
-		return encodeCreateFieldMessage(mt)
-	case *DeleteFieldMessage:
-		return encodeDeleteFieldMessage(mt)
-	case *CreateViewMessage:
-		return encodeCreateViewMessage(mt)
-	case *DeleteViewMessage:
-		return encodeDeleteViewMessage(mt)
-	case *ClusterStatus:
-		return encodeClusterStatus(mt)
-	case *ResizeInstruction:
-		return encodeResizeInstruction(mt)
-	case *ResizeInstructionComplete:
-		return encodeResizeInstructionComplete(mt)
-	case *SetCoordinatorMessage:
-		return encodeSetCoordinatorMessage(mt)
-	case *UpdateCoordinatorMessage:
-		return encodeUpdateCoordinatorMessage(mt)
-	case *NodeStateMessage:
-		return encodeNodeStateMessage(mt)
-	case *RecalculateCaches:
-		return encodeRecalculateCaches(mt)
-	case *NodeEvent:
-		return encodeNodeEventMessage(mt)
-	case *NodeStatus:
-		return encodeNodeStatus(mt)
+func getMessage(typ byte) Message {
+	switch typ {
+	case messageTypeCreateShard:
+		return &CreateShardMessage{}
+	case messageTypeCreateIndex:
+		return &CreateIndexMessage{}
+	case messageTypeDeleteIndex:
+		return &DeleteIndexMessage{}
+	case messageTypeCreateField:
+		return &CreateFieldMessage{}
+	case messageTypeDeleteField:
+		return &DeleteFieldMessage{}
+	case messageTypeCreateView:
+		return &CreateViewMessage{}
+	case messageTypeDeleteView:
+		return &DeleteViewMessage{}
+	case messageTypeClusterStatus:
+		return &ClusterStatus{}
+	case messageTypeResizeInstruction:
+		return &ResizeInstruction{}
+	case messageTypeResizeInstructionComplete:
+		return &ResizeInstructionComplete{}
+	case messageTypeSetCoordinator:
+		return &SetCoordinatorMessage{}
+	case messageTypeUpdateCoordinator:
+		return &UpdateCoordinatorMessage{}
+	case messageTypeNodeState:
+		return &NodeStateMessage{}
+	case messageTypeRecalculateCaches:
+		return &RecalculateCaches{}
+	case messageTypeNodeEvent:
+		return &NodeEvent{}
+	case messageTypeNodeStatus:
+		return &NodeStatus{}
+	default:
+		panic(fmt.Sprintf("unknown message type %d", typ))
+	}
+}
+
+func getMessageType(m Message) byte {
+	switch m.(type) {
+	case *CreateShardMessage:
+		return messageTypeCreateShard
+	case *CreateIndexMessage:
+		return messageTypeCreateIndex
+	case *DeleteIndexMessage:
+		return messageTypeDeleteIndex
+	case *CreateFieldMessage:
+		return messageTypeCreateField
+	case *DeleteFieldMessage:
+		return messageTypeDeleteField
+	case *CreateViewMessage:
+		return messageTypeCreateView
+	case *DeleteViewMessage:
+		return messageTypeDeleteView
+	case *ClusterStatus:
+		return messageTypeClusterStatus
+	case *ResizeInstruction:
+		return messageTypeResizeInstruction
+	case *ResizeInstructionComplete:
+		return messageTypeResizeInstructionComplete
+	case *SetCoordinatorMessage:
+		return messageTypeSetCoordinator
+	case *UpdateCoordinatorMessage:
+		return messageTypeUpdateCoordinator
+	case *NodeStateMessage:
+		return messageTypeNodeState
+	case *RecalculateCaches:
+		return messageTypeRecalculateCaches
+	case *NodeEvent:
+		return messageTypeNodeEvent
+	case *NodeStatus:
+		return messageTypeNodeStatus
+	default:
+		panic(fmt.Sprintf("don't have type for message %#v", m))
 	}
-	return nil
 }
 
 // UnmarshalMessage decodes the byte slice into a protobuf message.
diff --git a/client.go b/client.go
index 59e3ad59b..01e0b847a 100644
--- a/client.go
+++ b/client.go
@@ -4,7 +4,6 @@ import (
 	"context"
 	"io"
 
-	"github.com/gogo/protobuf/proto"
 	"github.com/pilosa/pilosa/internal"
 )
 
@@ -49,7 +48,7 @@ type InternalClient interface {
 	BlockData(ctx context.Context, uri *URI, index, field string, shard uint64, block int) ([]uint64, []uint64, error)
 	ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
 	RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
-	SendMessage(ctx context.Context, uri *URI, pb proto.Message) error
+	SendMessage(ctx context.Context, uri *URI, msg []byte) error
 	RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error)
 }
 
@@ -128,7 +127,7 @@ func (n NopInternalClient) ColumnAttrDiff(ctx context.Context, uri *URI, index s
 func (n NopInternalClient) RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
 	return nil, nil
 }
-func (n NopInternalClient) SendMessage(ctx context.Context, uri *URI, pb proto.Message) error {
+func (n NopInternalClient) SendMessage(ctx context.Context, uri *URI, msg []byte) error {
 	return nil
 }
 func (n NopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error) {
diff --git a/cluster.go b/cluster.go
index 2d25ebeaa..e2f29fde5 100644
--- a/cluster.go
+++ b/cluster.go
@@ -1890,10 +1890,10 @@ func decodeField(f *internal.Field) *FieldInfo {
 	fi := &FieldInfo{
 		Name:    f.Name,
 		Options: *decodeFieldOptions(f.Meta),
-		Views:   make([]*viewInfo, 0, len(f.Views)),
+		Views:   make([]*ViewInfo, 0, len(f.Views)),
 	}
 	for _, viewname := range f.Views {
-		fi.Views = append(fi.Views, &viewInfo{Name: viewname})
+		fi.Views = append(fi.Views, &ViewInfo{Name: viewname})
 	}
 	return fi
 }
diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go
new file mode 100644
index 000000000..1c83ba4ea
--- /dev/null
+++ b/encoding/proto/proto.go
@@ -0,0 +1,594 @@
+package proto
+
+import (
+	"fmt"
+
+	"github.com/gogo/protobuf/proto"
+	"github.com/pilosa/pilosa"
+	"github.com/pilosa/pilosa/internal"
+	"github.com/pkg/errors"
+)
+
+// Serializer implements pilosa.Serializer for protobufs.
+type Serializer struct{}
+
+// Marshal turns pilosa messages into protobuf serialized bytes.
+func (Serializer) Marshal(m pilosa.Message) ([]byte, error) {
+	pm := encodeToProto(m)
+	if pm == nil {
+		return nil, errors.New("passed invalid pilosa.Message")
+	}
+	buf, err := proto.Marshal(pm)
+	return buf, errors.Wrap(err, "marshalling")
+}
+
+// Unmarshal takes byte slices and protobuf deserializes them into a pilosa Message.
+func (Serializer) Unmarshal(buf []byte, m pilosa.Message) error {
+	switch mt := m.(type) {
+	case *pilosa.CreateShardMessage:
+		msg := &internal.CreateShardMessage{}
+		err := proto.Unmarshal(buf, msg)
+		if err != nil {
+			return errors.Wrap(err, "unmarshaling CreateShardMessage")
+		}
+		decodeCreateShardMessage(msg, mt)
+		return nil
+	case *pilosa.CreateIndexMessage:
+		msg := &internal.CreateIndexMessage{}
+		err := proto.Unmarshal(buf, msg)
+		if err != nil {
+			return errors.Wrap(err, "unmarshaling CreateIndexMessage")
+		}
+		decodeCreateIndexMessage(msg, mt)
+		return nil
+	case *pilosa.DeleteIndexMessage:
+		msg := &internal.DeleteIndexMessage{}
+		err := proto.Unmarshal(buf, msg)
+		if err != nil {
+			return errors.Wrap(err, "unmarshaling DeleteIndexMessage")
+		}
+		decodeDeleteIndexMessage(msg, mt)
+		return nil
+	case *pilosa.CreateFieldMessage:
+		msg := &internal.CreateFieldMessage{}
+		err := proto.Unmarshal(buf, msg)
+		if err != nil {
+			return errors.Wrap(err, "unmarshaling CreateFieldMessage")
+		}
+		decodeCreateFieldMessage(msg, mt)
+		return nil
+	case *pilosa.DeleteFieldMessage:
+		msg := &internal.DeleteFieldMessage{}
+		err := proto.Unmarshal(buf, msg)
+		if err != nil {
+			return errors.Wrap(err, "unmarshaling DeleteFieldMessage")
+		}
+		decodeDeleteFieldMessage(msg, mt)
+		return nil
+	case *pilosa.CreateViewMessage:
+		msg := &internal.CreateViewMessage{}
+		err := proto.Unmarshal(buf, msg)
+		if err != nil {
+			return errors.Wrap(err, "unmarshaling CreateViewMessage")
+		}
+		decodeCreateViewMessage(msg, mt)
+		return nil
+	case *pilosa.DeleteViewMessage:
+		msg := &internal.DeleteViewMessage{}
+		err := proto.Unmarshal(buf, msg)
+		if err != nil {
+			return errors.Wrap(err, "unmarshaling DeleteViewMessage")
+		}
+		decodeDeleteViewMessage(msg, mt)
+		return nil
+	case *pilosa.ClusterStatus:
+		msg := &internal.ClusterStatus{}
+		err := proto.Unmarshal(buf, msg)
+		if err != nil {
+			return errors.Wrap(err, "unmarshaling ClusterStatus")
+		}
+		decodeClusterStatus(msg, mt)
+		return nil
+	case *pilosa.ResizeInstruction:
+		msg := &internal.ResizeInstruction{}
+		err := proto.Unmarshal(buf, msg)
+		if err != nil {
+			return errors.Wrap(err, "unmarshaling ResizeInstruction")
+		}
+		decodeResizeInstruction(msg, mt)
+		return nil
+	case *pilosa.ResizeInstructionComplete:
+		msg := &internal.ResizeInstructionComplete{}
+		err := proto.Unmarshal(buf, msg)
+		if err != nil {
+			return errors.Wrap(err, "unmarshaling ResizeInstructionComplete")
+		}
+		decodeResizeInstructionComplete(msg, mt)
+		return nil
+	case *pilosa.SetCoordinatorMessage:
+		msg := &internal.SetCoordinatorMessage{}
+		err := proto.Unmarshal(buf, msg)
+		if err != nil {
+			return errors.Wrap(err, "unmarshaling SetCoordinatorMessage")
+		}
+		decodeSetCoordinatorMessage(msg, mt)
+		return nil
+	case *pilosa.UpdateCoordinatorMessage:
+		msg := &internal.UpdateCoordinatorMessage{}
+		err := proto.Unmarshal(buf, msg)
+		if err != nil {
+			return errors.Wrap(err, "unmarshaling UpdateCoordinatorMessage")
+		}
+		decodeUpdateCoordinatorMessage(msg, mt)
+		return nil
+	case *pilosa.NodeStateMessage:
+		msg := &internal.NodeStateMessage{}
+		err := proto.Unmarshal(buf, msg)
+		if err != nil {
+			return errors.Wrap(err, "unmarshaling NodeStateMessage")
+		}
+		decodeNodeStateMessage(msg, mt)
+		return nil
+	case *pilosa.RecalculateCaches:
+		msg := &internal.RecalculateCaches{}
+		err := proto.Unmarshal(buf, msg)
+		if err != nil {
+			return errors.Wrap(err, "unmarshaling RecalculateCaches")
+		}
+		decodeRecalculateCaches(msg, mt)
+		return nil
+	case *pilosa.NodeEvent:
+		msg := &internal.NodeEventMessage{}
+		err := proto.Unmarshal(buf, msg)
+		if err != nil {
+			return errors.Wrap(err, "unmarshaling NodeEvent")
+		}
+		decodeNodeEventMessage(msg, mt)
+		return nil
+	case *pilosa.NodeStatus:
+		msg := &internal.NodeStatus{}
+		err := proto.Unmarshal(buf, msg)
+		if err != nil {
+			return errors.Wrap(err, "unmarshaling NodeStatus")
+		}
+		decodeNodeStatus(msg, mt)
+		return nil
+	default:
+		panic(fmt.Sprintf("unhandled pilosa.Message of type %T: %#v", mt, m))
+	}
+}
+
+func encodeToProto(m pilosa.Message) proto.Message {
+	switch mt := m.(type) {
+	case *pilosa.CreateShardMessage:
+		return encodeCreateShardMessage(mt)
+	case *pilosa.CreateIndexMessage:
+		return encodeCreateIndexMessage(mt)
+	case *pilosa.DeleteIndexMessage:
+		return encodeDeleteIndexMessage(mt)
+	case *pilosa.CreateFieldMessage:
+		return encodeCreateFieldMessage(mt)
+	case *pilosa.DeleteFieldMessage:
+		return encodeDeleteFieldMessage(mt)
+	case *pilosa.CreateViewMessage:
+		return encodeCreateViewMessage(mt)
+	case *pilosa.DeleteViewMessage:
+		return encodeDeleteViewMessage(mt)
+	case *pilosa.ClusterStatus:
+		return encodeClusterStatus(mt)
+	case *pilosa.ResizeInstruction:
+		return encodeResizeInstruction(mt)
+	case *pilosa.ResizeInstructionComplete:
+		return encodeResizeInstructionComplete(mt)
+	case *pilosa.SetCoordinatorMessage:
+		return encodeSetCoordinatorMessage(mt)
+	case *pilosa.UpdateCoordinatorMessage:
+		return encodeUpdateCoordinatorMessage(mt)
+	case *pilosa.NodeStateMessage:
+		return encodeNodeStateMessage(mt)
+	case *pilosa.RecalculateCaches:
+		return encodeRecalculateCaches(mt)
+	case *pilosa.NodeEvent:
+		return encodeNodeEventMessage(mt)
+	case *pilosa.NodeStatus:
+		return encodeNodeStatus(mt)
+	}
+	return nil
+}
+
+func encodeResizeInstruction(m *pilosa.ResizeInstruction) *internal.ResizeInstruction {
+	return &internal.ResizeInstruction{
+		JobID:         m.JobID,
+		Node:          EncodeNode(m.Node),
+		Coordinator:   EncodeNode(m.Coordinator),
+		Sources:       encodeResizeSources(m.Sources),
+		Schema:        encodeSchema(m.Schema),
+		ClusterStatus: encodeClusterStatus(m.ClusterStatus),
+	}
+}
+
+func encodeResizeSources(srcs []*pilosa.ResizeSource) []*internal.ResizeSource {
+	new := make([]*internal.ResizeSource, 0, len(srcs))
+	for _, src := range srcs {
+		new = append(new, encodeResizeSource(src))
+	}
+	return new
+}
+
+func encodeResizeSource(m *pilosa.ResizeSource) *internal.ResizeSource {
+	return &internal.ResizeSource{
+		Node:  EncodeNode(m.Node),
+		Index: m.Index,
+		Field: m.Field,
+		View:  m.View,
+		Shard: m.Shard,
+	}
+}
+
+func encodeSchema(m *pilosa.Schema) *internal.Schema {
+	return &internal.Schema{
+		Indexes: encodeIndexInfos(m.Indexes),
+	}
+}
+
+func encodeIndexInfos(idxs []*pilosa.IndexInfo) []*internal.Index {
+	new := make([]*internal.Index, 0, len(idxs))
+	for _, idx := range idxs {
+		new = append(new, encodeIndexInfo(idx))
+	}
+	return new
+}
+
+func encodeIndexInfo(idx *pilosa.IndexInfo) *internal.Index {
+	return &internal.Index{
+		Name:   idx.Name,
+		Fields: encodeFieldInfos(idx.Fields),
+	}
+}
+
+func encodeFieldInfos(fs []*pilosa.FieldInfo) []*internal.Field {
+	new := make([]*internal.Field, 0, len(fs))
+	for _, f := range fs {
+		new = append(new, encodeFieldInfo(f))
+	}
+	return new
+}
+
+func encodeFieldInfo(f *pilosa.FieldInfo) *internal.Field {
+	ifield := &internal.Field{
+		Name:  f.Name,
+		Meta:  encodeFieldOptions(&f.Options),
+		Views: make([]string, 0, len(f.Views)),
+	}
+
+	for _, viewinfo := range f.Views {
+		ifield.Views = append(ifield.Views, viewinfo.Name)
+	}
+	return ifield
+}
+
+func encodeFieldOptions(o *pilosa.FieldOptions) *internal.FieldOptions {
+	if o == nil {
+		return nil
+	}
+	return &internal.FieldOptions{
+		Type:        o.Type,
+		CacheType:   o.CacheType,
+		CacheSize:   o.CacheSize,
+		Min:         o.Min,
+		Max:         o.Max,
+		TimeQuantum: string(o.TimeQuantum),
+		Keys:        o.Keys,
+	}
+}
+
+// EncodeNodes converts a slice of Nodes into its internal representation.
+func EncodeNodes(a []*pilosa.Node) []*internal.Node {
+	other := make([]*internal.Node, len(a))
+	for i := range a {
+		other[i] = EncodeNode(a[i])
+	}
+	return other
+}
+
+// EncodeNode converts a Node into its internal representation.
+func EncodeNode(n *pilosa.Node) *internal.Node {
+	return &internal.Node{
+		ID:            n.ID,
+		URI:           n.URI.Encode(),
+		IsCoordinator: n.IsCoordinator,
+	}
+}
+
+func encodeClusterStatus(m *pilosa.ClusterStatus) *internal.ClusterStatus {
+	return &internal.ClusterStatus{
+		State:     m.State,
+		ClusterID: m.ClusterID,
+		Nodes:     EncodeNodes(m.Nodes),
+	}
+}
+
+func encodeCreateShardMessage(m *pilosa.CreateShardMessage) *internal.CreateShardMessage {
+	return &internal.CreateShardMessage{
+		Index: m.Index,
+		Shard: m.Shard,
+	}
+}
+
+func encodeCreateIndexMessage(m *pilosa.CreateIndexMessage) *internal.CreateIndexMessage {
+	return &internal.CreateIndexMessage{
+		Index: m.Index,
+		Meta:  encodeIndexMeta(m.Meta),
+	}
+}
+
+func encodeIndexMeta(m *pilosa.IndexOptions) *internal.IndexMeta {
+	return &internal.IndexMeta{
+		Keys: m.Keys,
+	}
+}
+
+func encodeDeleteIndexMessage(m *pilosa.DeleteIndexMessage) *internal.DeleteIndexMessage {
+	return &internal.DeleteIndexMessage{
+		Index: m.Index,
+	}
+}
+
+func encodeCreateFieldMessage(m *pilosa.CreateFieldMessage) *internal.CreateFieldMessage {
+	return &internal.CreateFieldMessage{
+		Index: m.Index,
+		Field: m.Field,
+		Meta:  encodeFieldOptions(m.Meta),
+	}
+}
+
+func encodeDeleteFieldMessage(m *pilosa.DeleteFieldMessage) *internal.DeleteFieldMessage {
+	return &internal.DeleteFieldMessage{
+		Index: m.Index,
+		Field: m.Field,
+	}
+}
+
+func encodeCreateViewMessage(m *pilosa.CreateViewMessage) *internal.CreateViewMessage {
+	return &internal.CreateViewMessage{
+		Index: m.Index,
+		Field: m.Field,
+		View:  m.View,
+	}
+}
+
+func encodeDeleteViewMessage(m *pilosa.DeleteViewMessage) *internal.DeleteViewMessage {
+	return &internal.DeleteViewMessage{
+		Index: m.Index,
+		Field: m.Field,
+		View:  m.View,
+	}
+}
+
+func encodeResizeInstructionComplete(m *pilosa.ResizeInstructionComplete) *internal.ResizeInstructionComplete {
+	return &internal.ResizeInstructionComplete{
+		JobID: m.JobID,
+		Node:  EncodeNode(m.Node),
+		Error: m.Error,
+	}
+}
+
+func encodeSetCoordinatorMessage(m *pilosa.SetCoordinatorMessage) *internal.SetCoordinatorMessage {
+	return &internal.SetCoordinatorMessage{
+		New: EncodeNode(m.New),
+	}
+}
+
+func encodeUpdateCoordinatorMessage(m *pilosa.UpdateCoordinatorMessage) *internal.UpdateCoordinatorMessage {
+	return &internal.UpdateCoordinatorMessage{
+		New: EncodeNode(m.New),
+	}
+}
+
+func encodeNodeStateMessage(m *pilosa.NodeStateMessage) *internal.NodeStateMessage {
+	return &internal.NodeStateMessage{
+		NodeID: m.NodeID,
+		State:  m.State,
+	}
+}
+
+func encodeNodeEventMessage(m *pilosa.NodeEvent) *internal.NodeEventMessage {
+	return &internal.NodeEventMessage{
+		Event: uint32(m.Event),
+		Node:  EncodeNode(m.Node),
+	}
+}
+
+func encodeNodeStatus(m *pilosa.NodeStatus) *internal.NodeStatus {
+	return &internal.NodeStatus{
+		Node:      EncodeNode(m.Node),
+		MaxShards: &internal.MaxShards{Standard: m.MaxShards},
+		Schema:    encodeSchema(m.Schema),
+	}
+}
+
+func encodeRecalculateCaches(*pilosa.RecalculateCaches) *internal.RecalculateCaches {
+	return &internal.RecalculateCaches{}
+}
+
+func decodeResizeInstruction(ri *internal.ResizeInstruction, m *pilosa.ResizeInstruction) {
+	m.JobID = ri.JobID
+	m.Node = &pilosa.Node{}
+	decodeNode(ri.Node, m.Node)
+	m.Coordinator = &pilosa.Node{}
+	decodeNode(ri.Coordinator, m.Coordinator)
+	m.Sources = make([]*pilosa.ResizeSource, len(ri.Sources))
+	decodeResizeSources(ri.Sources, m.Sources)
+	m.Schema = &pilosa.Schema{}
+	decodeSchema(ri.Schema, m.Schema)
+	m.ClusterStatus = &pilosa.ClusterStatus{}
+	decodeClusterStatus(ri.ClusterStatus, m.ClusterStatus)
+}
+
+func decodeResizeSources(srcs []*internal.ResizeSource, m []*pilosa.ResizeSource) {
+	for i := range srcs {
+		m[i] = &pilosa.ResizeSource{}
+		decodeResizeSource(srcs[i], m[i])
+	}
+}
+
+func decodeResizeSource(rs *internal.ResizeSource, m *pilosa.ResizeSource) {
+	m.Node = &pilosa.Node{}
+	decodeNode(rs.Node, m.Node)
+	m.Index = rs.Index
+	m.Field = rs.Field
+	m.View = rs.View
+	m.Shard = rs.Shard
+}
+
+func decodeSchema(s *internal.Schema, m *pilosa.Schema) {
+	m.Indexes = make([]*pilosa.IndexInfo, len(s.Indexes))
+	decodeIndexes(s.Indexes, m.Indexes)
+}
+
+func decodeIndexes(idxs []*internal.Index, m []*pilosa.IndexInfo) {
+	for i := range idxs {
+		m[i] = &pilosa.IndexInfo{}
+		decodeIndex(idxs[i], m[i])
+	}
+}
+
+func decodeIndex(idx *internal.Index, m *pilosa.IndexInfo) {
+	m.Name = idx.Name
+	m.Fields = make([]*pilosa.FieldInfo, len(idx.Fields))
+	decodeFields(idx.Fields, m.Fields)
+}
+
+func decodeFields(fs []*internal.Field, m []*pilosa.FieldInfo) {
+	for i := range fs {
+		m[i] = &pilosa.FieldInfo{}
+		decodeField(fs[i], m[i])
+	}
+}
+
+func decodeField(f *internal.Field, m *pilosa.FieldInfo) {
+	m.Name = f.Name
+	m.Options = pilosa.FieldOptions{}
+	decodeFieldOptions(f.Meta, &m.Options)
+	m.Views = make([]*pilosa.ViewInfo, 0, len(f.Views))
+	for _, viewname := range f.Views {
+		m.Views = append(m.Views, &pilosa.ViewInfo{Name: viewname})
+	}
+}
+
+func decodeFieldOptions(options *internal.FieldOptions, m *pilosa.FieldOptions) {
+	m.Type = options.Type
+	m.CacheType = options.CacheType
+	m.CacheSize = options.CacheSize
+	m.Min = options.Min
+	m.Max = options.Max
+	m.TimeQuantum = pilosa.TimeQuantum(options.TimeQuantum)
+	m.Keys = options.Keys
+}
+
+func decodeNodes(a []*internal.Node, m []*pilosa.Node) {
+	for i := range a {
+		m[i] = &pilosa.Node{}
+		decodeNode(a[i], m[i])
+	}
+}
+
+func decodeClusterStatus(cs *internal.ClusterStatus, m *pilosa.ClusterStatus) {
+	m.State = cs.State
+	m.ClusterID = cs.ClusterID
+	m.Nodes = make([]*pilosa.Node, len(cs.Nodes))
+	decodeNodes(cs.Nodes, m.Nodes)
+}
+
+func decodeNode(node *internal.Node, m *pilosa.Node) {
+	m.ID = node.ID
+	decodeURI(node.URI, &m.URI)
+	m.IsCoordinator = node.IsCoordinator
+}
+
+func decodeURI(i *internal.URI, m *pilosa.URI) {
+	m.Scheme = i.Scheme
+	m.Host = i.Host
+	m.Port = uint16(i.Port)
+}
+
+func decodeCreateShardMessage(pb *internal.CreateShardMessage, m *pilosa.CreateShardMessage) {
+	m.Index = pb.Index
+	m.Shard = pb.Shard
+}
+
+func decodeCreateIndexMessage(pb *internal.CreateIndexMessage, m *pilosa.CreateIndexMessage) {
+	m.Index = pb.Index
+	m.Meta = &pilosa.IndexOptions{}
+	decodeIndexMeta(pb.Meta, m.Meta)
+}
+
+func decodeIndexMeta(pb *internal.IndexMeta, m *pilosa.IndexOptions) {
+	m.Keys = pb.Keys
+}
+
+func decodeDeleteIndexMessage(pb *internal.DeleteIndexMessage, m *pilosa.DeleteIndexMessage) {
+	m.Index = pb.Index
+}
+
+func decodeCreateFieldMessage(pb *internal.CreateFieldMessage, m *pilosa.CreateFieldMessage) {
+	m.Index = pb.Index
+	m.Field = pb.Field
+	m.Meta = &pilosa.FieldOptions{}
+	decodeFieldOptions(pb.Meta, m.Meta)
+}
+
+func decodeDeleteFieldMessage(pb *internal.DeleteFieldMessage, m *pilosa.DeleteFieldMessage) {
+	m.Index = pb.Index
+	m.Field = pb.Field
+}
+
+func decodeCreateViewMessage(pb *internal.CreateViewMessage, m *pilosa.CreateViewMessage) {
+	m.Index = pb.Index
+	m.Field = pb.Field
+	m.View = pb.View
+}
+
+func decodeDeleteViewMessage(pb *internal.DeleteViewMessage, m *pilosa.DeleteViewMessage) {
+	m.Index = pb.Index
+	m.Field = pb.Field
+	m.View = pb.View
+}
+
+func decodeResizeInstructionComplete(pb *internal.ResizeInstructionComplete, m *pilosa.ResizeInstructionComplete) {
+	m.JobID = pb.JobID
+	m.Node = &pilosa.Node{}
+	decodeNode(pb.Node, m.Node)
+	m.Error = pb.Error
+}
+
+func decodeSetCoordinatorMessage(pb *internal.SetCoordinatorMessage, m *pilosa.SetCoordinatorMessage) {
+	m.New = &pilosa.Node{}
+	decodeNode(pb.New, m.New)
+}
+
+func decodeUpdateCoordinatorMessage(pb *internal.UpdateCoordinatorMessage, m *pilosa.UpdateCoordinatorMessage) {
+	m.New = &pilosa.Node{}
+	decodeNode(pb.New, m.New)
+}
+
+func decodeNodeStateMessage(pb *internal.NodeStateMessage, m *pilosa.NodeStateMessage) {
+	m.NodeID = pb.NodeID
+	m.State = pb.State
+}
+
+func decodeNodeEventMessage(pb *internal.NodeEventMessage, m *pilosa.NodeEvent) {
+	m.Event = pilosa.NodeEventType(pb.Event)
+	m.Node = &pilosa.Node{}
+	decodeNode(pb.Node, m.Node)
+}
+
+func decodeNodeStatus(pb *internal.NodeStatus, m *pilosa.NodeStatus) {
+	m.Node = &pilosa.Node{}
+	decodeNode(pb.Node, m.Node)
+	m.MaxShards = pb.MaxShards.Standard
+	m.Schema = &pilosa.Schema{}
+	decodeSchema(pb.Schema, m.Schema)
+}
+
+func decodeRecalculateCaches(pb *internal.RecalculateCaches, m *pilosa.RecalculateCaches) {}
diff --git a/field.go b/field.go
index eea6bb10e..fb7e9af4f 100644
--- a/field.go
+++ b/field.go
@@ -1077,13 +1077,13 @@ func (f *Field) MarshalJSON() ([]byte, error) {
 	thing := struct {
 		Name    string
 		Options FieldOptions
-		Views   []*viewInfo
+		Views   []*ViewInfo
 	}{
 		Name:    f.Name(),
 		Options: f.Options(),
 	}
 	for _, viewname := range f.viewNames() {
-		thing.Views = append(thing.Views, &viewInfo{Name: viewname})
+		thing.Views = append(thing.Views, &ViewInfo{Name: viewname})
 	}
 	return json.Marshal(thing)
 }
@@ -1117,7 +1117,7 @@ func (p fieldSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() }
 type FieldInfo struct {
 	Name    string       `json:"name"`
 	Options FieldOptions `json:"options"`
-	Views   []*viewInfo  `json:"views,omitempty"`
+	Views   []*ViewInfo  `json:"views,omitempty"`
 }
 
 type fieldInfoSlice []*FieldInfo
diff --git a/gossip/gossip.go b/gossip/gossip.go
index 3dc7c202b..cd1d9ad0b 100644
--- a/gossip/gossip.go
+++ b/gossip/gossip.go
@@ -148,7 +148,7 @@ func WithLogger(logger *log.Logger) GossipMemberSetOption {
 
 // NewGossipMemberSet returns a new instance of GossipMemberSet based on options.
 func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...GossipMemberSetOption) (*GossipMemberSet, error) {
-	host := api.Node().URI.Host()
+	host := api.Node().URI.GetHost()
 	g := &GossipMemberSet{
 		papi:   api,
 		Logger: pilosa.NopLogger,
@@ -193,10 +193,10 @@ func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...GossipMemberSetO
 	conf := memberlist.DefaultWANConfig()
 	conf.Transport = g.transport.Net
 	conf.Name = api.Node().ID
-	conf.BindAddr = api.Node().URI.Host()
+	conf.BindAddr = api.Node().URI.GetHost()
 	conf.BindPort = port
 	conf.AdvertisePort = port
-	conf.AdvertiseAddr = hostToIP(api.Node().URI.Host())
+	conf.AdvertiseAddr = hostToIP(api.Node().URI.GetHost())
 	//
 	conf.TCPTimeout = time.Duration(cfg.StreamTimeout)
 	conf.SuspicionMult = cfg.SuspicionMult
diff --git a/holder.go b/holder.go
index 83ffd6967..10e147098 100644
--- a/holder.go
+++ b/holder.go
@@ -217,7 +217,7 @@ func (h *Holder) Schema() []*IndexInfo {
 		for _, field := range index.Fields() {
 			fi := &FieldInfo{Name: field.Name(), Options: field.Options()}
 			for _, view := range field.views() {
-				fi.Views = append(fi.Views, &viewInfo{Name: view.name})
+				fi.Views = append(fi.Views, &ViewInfo{Name: view.name})
 			}
 			sort.Sort(viewInfoSlice(fi.Views))
 			di.Fields = append(di.Fields, fi)
diff --git a/http/client.go b/http/client.go
index 27597a5cb..41602b778 100644
--- a/http/client.go
+++ b/http/client.go
@@ -31,6 +31,7 @@ import (
 
 	"github.com/gogo/protobuf/proto"
 	"github.com/pilosa/pilosa"
+	pilosaproto "github.com/pilosa/pilosa/encoding/proto"
 	"github.com/pilosa/pilosa/internal"
 	"github.com/pkg/errors"
 )
@@ -43,6 +44,7 @@ type ClientOptions struct {
 // InternalClient represents a client to the Pilosa cluster.
 type InternalClient struct {
 	defaultURI *pilosa.URI
+	serializer pilosa.Serializer
 
 	// The client to use for HTTP communication.
 	HTTPClient *http.Client
@@ -66,6 +68,7 @@ func NewInternalClient(host string, remoteClient *http.Client) (*InternalClient,
 func NewInternalClientFromURI(defaultURI *pilosa.URI, remoteClient *http.Client) *InternalClient {
 	return &InternalClient{
 		defaultURI: defaultURI,
+		serializer: pilosaproto.Serializer{},
 		HTTPClient: remoteClient,
 	}
 }
@@ -819,12 +822,7 @@ func (c *InternalClient) RowAttrDiff(ctx context.Context, uri *pilosa.URI, index
 }
 
 // SendMessage posts a message synchronously.
-func (c *InternalClient) SendMessage(ctx context.Context, uri *pilosa.URI, pb proto.Message) error {
-	msg, err := pilosa.MarshalMessage(pb)
-	if err != nil {
-		return fmt.Errorf("marshaling message: %v", err)
-	}
-
+func (c *InternalClient) SendMessage(ctx context.Context, uri *pilosa.URI, msg []byte) error {
 	u := uriPathToURL(uri, "/internal/cluster/message")
 	req, err := http.NewRequest("POST", u.String(), bytes.NewReader(msg))
 	if err != nil {
@@ -998,7 +996,7 @@ func pos(rowID, columnID uint64) uint64 {
 
 func uriPathToURL(uri *pilosa.URI, path string) url.URL {
 	return url.URL{
-		Scheme: uri.Scheme(),
+		Scheme: uri.GetScheme(),
 		Host:   uri.HostPort(),
 		Path:   path,
 	}
@@ -1006,7 +1004,7 @@ func uriPathToURL(uri *pilosa.URI, path string) url.URL {
 
 func nodePathToURL(node *pilosa.Node, path string) url.URL {
 	return url.URL{
-		Scheme: node.URI.Scheme(),
+		Scheme: node.URI.GetScheme(),
 		Host:   node.URI.HostPort(),
 		Path:   path,
 	}
diff --git a/server.go b/server.go
index b57935acb..b3bc6e224 100644
--- a/server.go
+++ b/server.go
@@ -54,6 +54,7 @@ type Server struct {
 	executor        *executor
 	hosts           []string
 	clusterDisabled bool
+	serializer      Serializer
 
 	// External
 	systemInfo SystemInfo
@@ -200,6 +201,13 @@ func OptServerClusterDisabled(disabled bool, hosts []string) ServerOption {
 	}
 }
 
+func OptServerSerializer(ser Serializer) ServerOption {
+	return func(s *Server) error {
+		s.serializer = ser
+		return nil
+	}
+}
+
 func OptServerIsCoordinator(is bool) ServerOption {
 	return func(s *Server) error {
 		s.isCoordinator = is
@@ -517,8 +525,12 @@ func (s *Server) receiveMessage(m Message) error {
 
 // SendSync represents an implementation of Broadcaster.
 func (s *Server) SendSync(m Message) error {
-	pb := encode(m)
 	var eg errgroup.Group
+	msg, err := s.serializer.Marshal(m)
+	if err != nil {
+		return fmt.Errorf("marshaling message: %v", err)
+	}
+	msg = append([]byte{getMessageType(m)}, msg...)
 	for _, node := range s.cluster.Nodes {
 		node := node
 		s.logger.Printf("SendSync to: %s", node.URI)
@@ -528,7 +540,7 @@ func (s *Server) SendSync(m Message) error {
 		}
 
 		eg.Go(func() error {
-			return s.defaultClient.SendMessage(context.Background(), &node.URI, pb)
+			return s.defaultClient.SendMessage(context.Background(), &node.URI, msg)
 		})
 	}
 
@@ -542,9 +554,13 @@ func (s *Server) SendAsync(m Message) error {
 
 // SendTo represents an implementation of Broadcaster.
 func (s *Server) SendTo(to *Node, m Message) error {
-	pb := encode(m)
 	s.logger.Printf("SendTo: %s", to.URI)
-	return s.defaultClient.SendMessage(context.Background(), &to.URI, pb)
+	msg, err := s.serializer.Marshal(m)
+	if err != nil {
+		return fmt.Errorf("marshaling message: %v", err)
+	}
+	msg = append([]byte{getMessageType(m)}, msg...)
+	return s.defaultClient.SendMessage(context.Background(), &to.URI, msg)
 }
 
 // node returns the pilosa.node object. It is used by membership protocols to
@@ -613,7 +629,7 @@ func (s *Server) monitorDiagnostics() {
 
 	s.diagnostics.Logger = s.logger
 	s.diagnostics.SetVersion(Version)
-	s.diagnostics.Set("Host", s.uri.host)
+	s.diagnostics.Set("Host", s.uri.Host)
 	s.diagnostics.Set("Cluster", strings.Join(s.cluster.nodeIDs(), ","))
 	s.diagnostics.Set("NumNodes", len(s.cluster.Nodes))
 	s.diagnostics.Set("NumCPU", runtime.NumCPU())
diff --git a/server/server.go b/server/server.go
index c8de0664c..f46f6a64a 100644
--- a/server/server.go
+++ b/server/server.go
@@ -35,6 +35,7 @@ import (
 
 	"github.com/pilosa/pilosa"
 	"github.com/pilosa/pilosa/boltdb"
+	"github.com/pilosa/pilosa/encoding/proto"
 	"github.com/pilosa/pilosa/gcnotify"
 	"github.com/pilosa/pilosa/gopsutil"
 	"github.com/pilosa/pilosa/gossip"
@@ -202,7 +203,7 @@ func (m *Command) SetupServer() error {
 
 	// Setup TLS
 	var TLSConfig *tls.Config
-	if uri.Scheme() == "https" {
+	if uri.GetScheme() == "https" {
 		if m.Config.TLS.CertificatePath == "" {
 			return errors.New("certificate path is required for TLS sockets")
 		}
@@ -235,7 +236,7 @@ func (m *Command) SetupServer() error {
 	}
 
 	// If port is 0, get auto-allocated port from listener
-	if uri.Port() == 0 {
+	if uri.GetPort() == 0 {
 		uri.SetPort(uint16(m.ln.Addr().(*net.TCPAddr).Port))
 	}
 
@@ -271,6 +272,7 @@ func (m *Command) SetupServer() error {
 		pilosa.OptServerInternalClient(http.NewInternalClientFromURI(uri, c)),
 		pilosa.OptServerPrimaryTranslateStore(primaryTranslateStore),
 		pilosa.OptServerClusterDisabled(m.Config.Cluster.Disabled, m.Config.Cluster.Hosts),
+		pilosa.OptServerSerializer(proto.Serializer{}),
 		coordinatorOpt,
 	}
 
@@ -309,7 +311,7 @@ func (m *Command) SetupNetworking() error {
 	}
 
 	// get the host portion of addr to use for binding
-	gossipHost := m.API.Node().URI.Host()
+	gossipHost := m.API.Node().URI.GetHost()
 	m.gossipTransport, err = gossip.NewTransport(gossipHost, gossipPort, m.logger.Logger())
 	if err != nil {
 		return errors.Wrap(err, "getting transport")
@@ -366,19 +368,19 @@ func NewStatsClient(name string, host string) (pilosa.StatsClient, error) {
 // getListener gets a net.Listener based on the config.
 func getListener(uri pilosa.URI, tlsconf *tls.Config) (ln net.Listener, err error) {
 	// If bind URI has the https scheme, enable TLS
-	if uri.Scheme() == "https" && tlsconf != nil {
+	if uri.GetScheme() == "https" && tlsconf != nil {
 		ln, err = tls.Listen("tcp", uri.HostPort(), tlsconf)
 		if err != nil {
 			return nil, errors.Wrap(err, "tls.Listener")
 		}
-	} else if uri.Scheme() == "http" {
+	} else if uri.GetScheme() == "http" {
 		// Open HTTP listener to determine port (if specified as :0).
 		ln, err = net.Listen("tcp", uri.HostPort())
 		if err != nil {
 			return nil, errors.Wrap(err, "net.Listen")
 		}
 	} else {
-		return nil, errors.Errorf("unsupported scheme: %s", uri.Scheme())
+		return nil, errors.Errorf("unsupported scheme: %s", uri.GetScheme())
 	}
 
 	return ln, nil
diff --git a/uri.go b/uri.go
index 5d823d8b6..2058f70fa 100644
--- a/uri.go
+++ b/uri.go
@@ -43,17 +43,17 @@ var addressRegexp = regexp.MustCompile(`^(([+a-z]+):\/\/)?([0-9a-z.-]+|\[[:0-9a-
 // 	localhost
 // 	:10101
 type URI struct {
-	scheme string `json:"scheme"`
-	host   string `json:"host"`
-	port   uint16 `json:"port"`
+	Scheme string `json:"scheme"`
+	Host   string `json:"host"`
+	Port   uint16 `json:"port"`
 }
 
 // DefaultURI creates and returns the default URI.
 func DefaultURI() *URI {
 	return &URI{
-		scheme: "http",
-		host:   "localhost",
-		port:   10101,
+		Scheme: "http",
+		Host:   "localhost",
+		Port:   10101,
 	}
 }
 
@@ -83,9 +83,9 @@ func NewURIFromAddress(address string) (*URI, error) {
 	return parseAddress(address)
 }
 
-// Scheme returns the scheme of this URI.
-func (u *URI) Scheme() string {
-	return u.scheme
+// GetScheme returns the scheme of this URI.
+func (u *URI) GetScheme() string {
+	return u.Scheme
 }
 
 // SetScheme sets the scheme of this URI.
@@ -94,13 +94,13 @@ func (u *URI) SetScheme(scheme string) error {
 	if m == nil {
 		return errors.New("invalid scheme")
 	}
-	u.scheme = scheme
+	u.Scheme = scheme
 	return nil
 }
 
-// Host returns the host of this URI.
-func (u *URI) Host() string {
-	return u.host
+// GetHost returns the host of this URI.
+func (u *URI) GetHost() string {
+	return u.Host
 }
 
 // SetHost sets the host of this URI.
@@ -109,18 +109,18 @@ func (u *URI) SetHost(host string) error {
 	if m == nil {
 		return errors.New("invalid host")
 	}
-	u.host = host
+	u.Host = host
 	return nil
 }
 
-// Port returns the port of this URI.
-func (u *URI) Port() uint16 {
-	return u.port
+// GetPort returns the port of this URI.
+func (u *URI) GetPort() uint16 {
+	return u.Port
 }
 
 // SetPort sets the port of this URI.
 func (u *URI) SetPort(port uint16) {
-	u.port = port
+	u.Port = port
 }
 
 // HostPort returns `Host:Port`
@@ -129,23 +129,23 @@ func (u *URI) HostPort() string {
 	if u == nil {
 		return ""
 	}
-	s := fmt.Sprintf("%s:%d", u.host, u.port)
+	s := fmt.Sprintf("%s:%d", u.Host, u.Port)
 	return s
 }
 
 // Normalize returns the address in a form usable by a HTTP client.
 func (u *URI) Normalize() string {
-	scheme := u.scheme
+	scheme := u.Scheme
 	index := strings.Index(scheme, "+")
 	if index >= 0 {
 		scheme = scheme[:index]
 	}
-	return fmt.Sprintf("%s://%s:%d", scheme, u.host, u.port)
+	return fmt.Sprintf("%s://%s:%d", scheme, u.Host, u.Port)
 }
 
 // String returns the address as a string.
 func (u URI) String() string {
-	return fmt.Sprintf("%s://%s:%d", u.scheme, u.host, u.port)
+	return fmt.Sprintf("%s://%s:%d", u.Scheme, u.Host, u.Port)
 }
 
 // Equals returns true if the checked URI is equivalent to this URI.
@@ -199,9 +199,9 @@ func parseAddress(address string) (uri *URI, err error) {
 		}
 	}
 	uri = &URI{
-		scheme: scheme,
-		host:   host,
-		port:   uint16(port),
+		Scheme: scheme,
+		Host:   host,
+		Port:   uint16(port),
 	}
 	return uri, nil
 }
@@ -213,9 +213,9 @@ func (u URI) Encode() *internal.URI {
 
 func encodeURI(u URI) *internal.URI {
 	return &internal.URI{
-		Scheme: u.scheme,
-		Host:   u.host,
-		Port:   uint32(u.port),
+		Scheme: u.Scheme,
+		Host:   u.Host,
+		Port:   uint32(u.Port),
 	}
 }
 
@@ -228,9 +228,9 @@ func decodeURI(i *internal.URI) URI {
 		return URI{}
 	}
 	return URI{
-		scheme: i.Scheme,
-		host:   i.Host,
-		port:   uint16(i.Port),
+		Scheme: i.Scheme,
+		Host:   i.Host,
+		Port:   uint16(i.Port),
 	}
 }
 
@@ -241,9 +241,9 @@ func (u *URI) MarshalJSON() ([]byte, error) {
 		Host   string `json:"host,omitempty"`
 		Port   uint16 `json:"port,omitempty"`
 	}
-	output.Scheme = u.scheme
-	output.Host = u.host
-	output.Port = u.port
+	output.Scheme = u.Scheme
+	output.Host = u.Host
+	output.Port = u.Port
 
 	return json.Marshal(output)
 }
@@ -257,8 +257,8 @@ func (u *URI) UnmarshalJSON(b []byte) error {
 	if err := json.Unmarshal(b, &input); err != nil {
 		return err
 	}
-	u.scheme = input.Scheme
-	u.host = input.Host
-	u.port = input.Port
+	u.Scheme = input.Scheme
+	u.Host = input.Host
+	u.Port = input.Port
 	return nil
 }
diff --git a/uri_internal_test.go b/uri_internal_test.go
index dbcbfa04d..2aac9651f 100644
--- a/uri_internal_test.go
+++ b/uri_internal_test.go
@@ -93,8 +93,8 @@ func TestSetScheme(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if uri.Scheme() != target {
-		t.Fatalf("%s != %s", uri.Scheme(), target)
+	if uri.GetScheme() != target {
+		t.Fatalf("%s != %s", uri.GetScheme(), target)
 	}
 }
 
@@ -105,8 +105,8 @@ func TestSetHost(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if uri.Host() != target {
-		t.Fatalf("%s != %s", uri.host, target)
+	if uri.GetHost() != target {
+		t.Fatalf("%s != %s", uri.Host, target)
 	}
 }
 
@@ -114,8 +114,8 @@ func TestSetPort(t *testing.T) {
 	uri := DefaultURI()
 	target := uint16(9999)
 	uri.SetPort(target)
-	if uri.Port() != target {
-		t.Fatalf("%d != %d", uri.port, target)
+	if uri.GetPort() != target {
+		t.Fatalf("%d != %d", uri.Port, target)
 	}
 }
 
@@ -147,14 +147,14 @@ func TestHostPort(t *testing.T) {
 }
 
 func compare(t *testing.T, uri *URI, scheme string, host string, port uint16) {
-	if uri.Scheme() != scheme {
-		t.Fatalf("Scheme does not match: %s != %s", uri.scheme, scheme)
+	if uri.GetScheme() != scheme {
+		t.Fatalf("Scheme does not match: %s != %s", uri.Scheme, scheme)
 	}
-	if uri.Host() != host {
-		t.Fatalf("Host does not match: %s != %s", uri.host, host)
+	if uri.GetHost() != host {
+		t.Fatalf("Host does not match: %s != %s", uri.Host, host)
 	}
-	if uri.Port() != port {
-		t.Fatalf("Port does not match: %d != %d", uri.port, port)
+	if uri.GetPort() != port {
+		t.Fatalf("Port does not match: %d != %d", uri.Port, port)
 	}
 }
 
diff --git a/view.go b/view.go
index 0f2e189cd..609664304 100644
--- a/view.go
+++ b/view.go
@@ -421,12 +421,12 @@ func (v *view) rangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*
 	return r, nil
 }
 
-// viewInfo represents schema information for a view.
-type viewInfo struct {
+// ViewInfo represents schema information for a view.
+type ViewInfo struct {
 	Name string `json:"name"`
 }
 
-type viewInfoSlice []*viewInfo
+type viewInfoSlice []*ViewInfo
 
 func (p viewInfoSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
 func (p viewInfoSlice) Len() int           { return len(p) }

From 6309d3b7f79a3b2a222e2feb30e754076aa82f7d Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Thu, 5 Jul 2018 12:32:49 -0500
Subject: [PATCH 15/32] get gossip using serializer stuff, remove proto and
 internal

---
 api.go                  |  4 ++++
 broadcast.go            | 48 +++++---------------------------------
 broadcast_test.go       | 51 -----------------------------------------
 encoding/proto/proto.go | 32 +++++++++++++++++---------
 gossip/gossip.go        | 49 ++++++++++++++++++++-------------------
 5 files changed, 57 insertions(+), 127 deletions(-)
 delete mode 100644 broadcast_test.go

diff --git a/api.go b/api.go
index 09f0531d5..cae91b20f 100644
--- a/api.go
+++ b/api.go
@@ -149,6 +149,10 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er
 	return resp, nil
 }
 
+func (api *API) Holder() *Holder {
+	return api.server.Holder()
+}
+
 // readColumnAttrSets returns a list of column attribute objects by id.
 func (api *API) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet, error) {
 	if index == nil {
diff --git a/broadcast.go b/broadcast.go
index 1da6ca57f..00835db64 100644
--- a/broadcast.go
+++ b/broadcast.go
@@ -16,7 +16,6 @@ package pilosa
 
 import (
 	"fmt"
-	"reflect"
 
 	"github.com/gogo/protobuf/proto"
 	"github.com/pilosa/pilosa/internal"
@@ -78,48 +77,13 @@ const (
 	messageTypeNodeStatus
 )
 
-// MarshalMessage encodes the protobuf message into a byte slice.
-func MarshalMessage(m proto.Message) ([]byte, error) {
-	var typ uint8
-	switch obj := m.(type) {
-	case *internal.CreateShardMessage:
-		typ = messageTypeCreateShard
-	case *internal.CreateIndexMessage:
-		typ = messageTypeCreateIndex
-	case *internal.DeleteIndexMessage:
-		typ = messageTypeDeleteIndex
-	case *internal.CreateFieldMessage:
-		typ = messageTypeCreateField
-	case *internal.DeleteFieldMessage:
-		typ = messageTypeDeleteField
-	case *internal.CreateViewMessage:
-		typ = messageTypeCreateView
-	case *internal.DeleteViewMessage:
-		typ = messageTypeDeleteView
-	case *internal.ClusterStatus:
-		typ = messageTypeClusterStatus
-	case *internal.ResizeInstruction:
-		typ = messageTypeResizeInstruction
-	case *internal.ResizeInstructionComplete:
-		typ = messageTypeResizeInstructionComplete
-	case *internal.SetCoordinatorMessage:
-		typ = messageTypeSetCoordinator
-	case *internal.UpdateCoordinatorMessage:
-		typ = messageTypeUpdateCoordinator
-	case *internal.NodeStateMessage:
-		typ = messageTypeNodeState
-	case *internal.RecalculateCaches:
-		typ = messageTypeRecalculateCaches
-	case *internal.NodeEventMessage:
-		typ = messageTypeNodeEvent
-	case *internal.NodeStatus:
-		typ = messageTypeNodeStatus
-	default:
-		return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj))
-	}
-	buf, err := proto.Marshal(m)
+// MarshalInternalMessage serializes the pilosa message and adds pilosa internal
+// type info which is used by the internal messaging stuff.
+func MarshalInternalMessage(m Message, s Serializer) ([]byte, error) {
+	typ := getMessageType(m)
+	buf, err := s.Marshal(m)
 	if err != nil {
-		return nil, errors.Wrap(err, "marshalling")
+		return nil, errors.Wrap(err, "marshaling")
 	}
 	return append([]byte{typ}, buf...), nil
 }
diff --git a/broadcast_test.go b/broadcast_test.go
deleted file mode 100644
index 415228718..000000000
--- a/broadcast_test.go
+++ /dev/null
@@ -1,51 +0,0 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package pilosa_test
-
-import (
-	"reflect"
-	"testing"
-
-	"github.com/gogo/protobuf/proto"
-	"github.com/pilosa/pilosa"
-	"github.com/pilosa/pilosa/internal"
-)
-
-// Ensure a message can be marshaled and unmarshaled.
-func TestMessage_Marshal(t *testing.T) {
-
-	testMessageMarshal(t, &internal.CreateShardMessage{
-		Index: "i",
-		Shard: 8,
-	})
-
-	testMessageMarshal(t, &internal.DeleteIndexMessage{
-		Index: "i",
-	})
-}
-
-func testMessageMarshal(t *testing.T, m proto.Message) {
-	marshalled, err := pilosa.MarshalMessage(m)
-	if err != nil {
-		t.Fatal(err)
-	}
-	unmarshalled, err := pilosa.UnmarshalMessage(marshalled)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual(unmarshalled, m) {
-		t.Fatalf("unexpected message marshalling: %s", unmarshalled)
-	}
-}
diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go
index 1c83ba4ea..10cc13dc4 100644
--- a/encoding/proto/proto.go
+++ b/encoding/proto/proto.go
@@ -153,6 +153,14 @@ func (Serializer) Unmarshal(buf []byte, m pilosa.Message) error {
 		}
 		decodeNodeStatus(msg, mt)
 		return nil
+	case *pilosa.Node:
+		msg := &internal.Node{}
+		err := proto.Unmarshal(buf, msg)
+		if err != nil {
+			return errors.Wrap(err, "unmarshaling Node")
+		}
+		decodeNode(msg, mt)
+		return nil
 	default:
 		panic(fmt.Sprintf("unhandled pilosa.Message of type %T: %#v", mt, m))
 	}
@@ -192,6 +200,8 @@ func encodeToProto(m pilosa.Message) proto.Message {
 		return encodeNodeEventMessage(mt)
 	case *pilosa.NodeStatus:
 		return encodeNodeStatus(mt)
+	case *pilosa.Node:
+		return encodeNode(mt)
 	}
 	return nil
 }
@@ -199,8 +209,8 @@ func encodeToProto(m pilosa.Message) proto.Message {
 func encodeResizeInstruction(m *pilosa.ResizeInstruction) *internal.ResizeInstruction {
 	return &internal.ResizeInstruction{
 		JobID:         m.JobID,
-		Node:          EncodeNode(m.Node),
-		Coordinator:   EncodeNode(m.Coordinator),
+		Node:          encodeNode(m.Node),
+		Coordinator:   encodeNode(m.Coordinator),
 		Sources:       encodeResizeSources(m.Sources),
 		Schema:        encodeSchema(m.Schema),
 		ClusterStatus: encodeClusterStatus(m.ClusterStatus),
@@ -217,7 +227,7 @@ func encodeResizeSources(srcs []*pilosa.ResizeSource) []*internal.ResizeSource {
 
 func encodeResizeSource(m *pilosa.ResizeSource) *internal.ResizeSource {
 	return &internal.ResizeSource{
-		Node:  EncodeNode(m.Node),
+		Node:  encodeNode(m.Node),
 		Index: m.Index,
 		Field: m.Field,
 		View:  m.View,
@@ -286,13 +296,13 @@ func encodeFieldOptions(o *pilosa.FieldOptions) *internal.FieldOptions {
 func EncodeNodes(a []*pilosa.Node) []*internal.Node {
 	other := make([]*internal.Node, len(a))
 	for i := range a {
-		other[i] = EncodeNode(a[i])
+		other[i] = encodeNode(a[i])
 	}
 	return other
 }
 
-// EncodeNode converts a Node into its internal representation.
-func EncodeNode(n *pilosa.Node) *internal.Node {
+// encodeNode converts a Node into its internal representation.
+func encodeNode(n *pilosa.Node) *internal.Node {
 	return &internal.Node{
 		ID:            n.ID,
 		URI:           n.URI.Encode(),
@@ -368,20 +378,20 @@ func encodeDeleteViewMessage(m *pilosa.DeleteViewMessage) *internal.DeleteViewMe
 func encodeResizeInstructionComplete(m *pilosa.ResizeInstructionComplete) *internal.ResizeInstructionComplete {
 	return &internal.ResizeInstructionComplete{
 		JobID: m.JobID,
-		Node:  EncodeNode(m.Node),
+		Node:  encodeNode(m.Node),
 		Error: m.Error,
 	}
 }
 
 func encodeSetCoordinatorMessage(m *pilosa.SetCoordinatorMessage) *internal.SetCoordinatorMessage {
 	return &internal.SetCoordinatorMessage{
-		New: EncodeNode(m.New),
+		New: encodeNode(m.New),
 	}
 }
 
 func encodeUpdateCoordinatorMessage(m *pilosa.UpdateCoordinatorMessage) *internal.UpdateCoordinatorMessage {
 	return &internal.UpdateCoordinatorMessage{
-		New: EncodeNode(m.New),
+		New: encodeNode(m.New),
 	}
 }
 
@@ -395,13 +405,13 @@ func encodeNodeStateMessage(m *pilosa.NodeStateMessage) *internal.NodeStateMessa
 func encodeNodeEventMessage(m *pilosa.NodeEvent) *internal.NodeEventMessage {
 	return &internal.NodeEventMessage{
 		Event: uint32(m.Event),
-		Node:  EncodeNode(m.Node),
+		Node:  encodeNode(m.Node),
 	}
 }
 
 func encodeNodeStatus(m *pilosa.NodeStatus) *internal.NodeStatus {
 	return &internal.NodeStatus{
-		Node:      EncodeNode(m.Node),
+		Node:      encodeNode(m.Node),
 		MaxShards: &internal.MaxShards{Standard: m.MaxShards},
 		Schema:    encodeSchema(m.Schema),
 	}
diff --git a/gossip/gossip.go b/gossip/gossip.go
index cd1d9ad0b..251851077 100644
--- a/gossip/gossip.go
+++ b/gossip/gossip.go
@@ -26,10 +26,9 @@ import (
 	"sync"
 	"time"
 
-	"github.com/gogo/protobuf/proto"
 	"github.com/hashicorp/memberlist"
 	"github.com/pilosa/pilosa"
-	"github.com/pilosa/pilosa/internal"
+	"github.com/pilosa/pilosa/encoding/proto"
 	"github.com/pilosa/pilosa/toml"
 	"github.com/pkg/errors"
 )
@@ -44,8 +43,9 @@ type GossipMemberSet struct {
 
 	broadcasts *memberlist.TransmitLimitedQueue
 
-	papi   *pilosa.API
-	config *gossipConfig
+	papi       *pilosa.API
+	serializer pilosa.Serializer
+	config     *gossipConfig
 
 	Logger pilosa.Logger
 
@@ -150,8 +150,9 @@ func WithLogger(logger *log.Logger) GossipMemberSetOption {
 func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...GossipMemberSetOption) (*GossipMemberSet, error) {
 	host := api.Node().URI.GetHost()
 	g := &GossipMemberSet{
-		papi:   api,
-		Logger: pilosa.NopLogger,
+		papi:       api,
+		serializer: proto.Serializer{},
+		Logger:     pilosa.NopLogger,
 	}
 
 	// options
@@ -222,7 +223,7 @@ func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...GossipMemberSetO
 
 // NodeMeta implementation of the memberlist.Delegate interface.
 func (g *GossipMemberSet) NodeMeta(limit int) []byte {
-	buf, err := proto.Marshal(pilosa.EncodeNode(g.papi.Node()))
+	buf, err := g.serializer.Marshal(g.papi.Node())
 	if err != nil {
 		g.Logger.Printf("marshal message error: %s", err)
 		return []byte{}
@@ -248,14 +249,14 @@ func (g *GossipMemberSet) GetBroadcasts(overhead, limit int) [][]byte {
 // LocalState implementation of the memberlist.Delegate interface
 // sends this Node's state data.
 func (g *GossipMemberSet) LocalState(join bool) []byte {
-	pb := &internal.NodeStatus{
-		Node:      pilosa.EncodeNode(g.papi.Node()),
-		MaxShards: &internal.MaxShards{Standard: g.papi.MaxShards(context.Background())},
-		Schema:    &internal.Schema{Indexes: pilosa.EncodeIndexes(g.papi.Schema(context.Background()))},
+	m := &pilosa.NodeStatus{
+		Node:      g.papi.Node(),
+		MaxShards: g.papi.MaxShards(context.Background()),
+		Schema:    &pilosa.Schema{Indexes: g.papi.Holder().Schema()},
 	}
 
 	// Marshal nodestate data to bytes.
-	buf, err := pilosa.MarshalMessage(pb)
+	buf, err := pilosa.MarshalInternalMessage(m, g.serializer)
 	if err != nil {
 		g.Logger.Printf("error marshalling nodestate data, err=%s", err)
 		return []byte{}
@@ -278,8 +279,9 @@ func (g *GossipMemberSet) MergeRemoteState(buf []byte, join bool) {
 // Care must be taken that events are processed in a timely manner from
 // the channel, since this delegate will block until an event can be sent.
 type gossipEventReceiver struct {
-	ch   chan memberlist.NodeEvent
-	papi *pilosa.API
+	ch         chan memberlist.NodeEvent
+	papi       *pilosa.API
+	serializer pilosa.Serializer
 
 	logger *log.Logger
 }
@@ -287,9 +289,10 @@ type gossipEventReceiver struct {
 // newGossipEventReceiver returns a new instance of GossipEventReceiver.
 func newGossipEventReceiver(logger *log.Logger, papi *pilosa.API) *gossipEventReceiver {
 	ger := &gossipEventReceiver{
-		ch:     make(chan memberlist.NodeEvent, 1),
-		logger: logger,
-		papi:   papi,
+		ch:         make(chan memberlist.NodeEvent, 1),
+		logger:     logger,
+		papi:       papi,
+		serializer: proto.Serializer{},
 	}
 	go ger.listen()
 	return ger
@@ -323,16 +326,16 @@ func (g *gossipEventReceiver) listen() {
 		}
 
 		// Get the node from the event.Node meta data.
-		var n internal.Node
-		if err := proto.Unmarshal(e.Node.Meta, &n); err != nil {
-			panic("failed to unmarshal event node meta data")
+		var n pilosa.Node
+		if err := g.serializer.Unmarshal(e.Node.Meta, &n); err != nil {
+			panic("failed to unmarshal event node meta into node")
 		}
 
-		ne := &internal.NodeEventMessage{
-			Event: uint32(nodeEventType),
+		ne := &pilosa.NodeEvent{
+			Event: nodeEventType,
 			Node:  &n,
 		}
-		buf, err := pilosa.MarshalMessage(ne)
+		buf, err := pilosa.MarshalInternalMessage(ne, g.serializer)
 		if err != nil {
 			panic(err)
 		}

From 59e80f9692aa2c687cd9365e61af6802bf8fccb6 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Thu, 5 Jul 2018 13:35:14 -0500
Subject: [PATCH 16/32] put Serializer on API, add QueryRequest/Response to
 serializer

---
 api.go                  |   3 +
 encoding/proto/proto.go | 288 ++++++++++++++++++++++++++++++++++++++++
 gossip/gossip.go        |  31 ++---
 http/handler.go         |  13 +-
 4 files changed, 310 insertions(+), 25 deletions(-)

diff --git a/api.go b/api.go
index cae91b20f..a9bd02f2d 100644
--- a/api.go
+++ b/api.go
@@ -38,6 +38,8 @@ type API struct {
 	holder  *Holder
 	cluster *cluster
 	server  *Server
+
+	Serializer Serializer
 }
 
 // APIOption is a functional option type for pilosa.API
@@ -48,6 +50,7 @@ func OptAPIServer(s *Server) APIOption {
 		a.server = s
 		a.holder = s.holder
 		a.cluster = s.cluster
+		a.Serializer = s.serializer
 		return nil
 	}
 }
diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go
index 10cc13dc4..0f2574f1b 100644
--- a/encoding/proto/proto.go
+++ b/encoding/proto/proto.go
@@ -2,6 +2,7 @@ package proto
 
 import (
 	"fmt"
+	"sort"
 
 	"github.com/gogo/protobuf/proto"
 	"github.com/pilosa/pilosa"
@@ -161,6 +162,23 @@ func (Serializer) Unmarshal(buf []byte, m pilosa.Message) error {
 		}
 		decodeNode(msg, mt)
 		return nil
+	case *pilosa.QueryRequest:
+		msg := &internal.QueryRequest{}
+		err := proto.Unmarshal(buf, msg)
+		if err != nil {
+			return errors.Wrap(err, "unmarshaling QueryRequest")
+		}
+		decodeQueryRequest(msg, mt)
+		return nil
+	case *pilosa.QueryResponse:
+		msg := &internal.QueryResponse{}
+		err := proto.Unmarshal(buf, msg)
+		if err != nil {
+			return errors.Wrap(err, "unmarshaling QueryResponse")
+		}
+		decodeQueryResponse(msg, mt)
+		return nil
+
 	default:
 		panic(fmt.Sprintf("unhandled pilosa.Message of type %T: %#v", mt, m))
 	}
@@ -202,10 +220,62 @@ func encodeToProto(m pilosa.Message) proto.Message {
 		return encodeNodeStatus(mt)
 	case *pilosa.Node:
 		return encodeNode(mt)
+	case *pilosa.QueryRequest:
+		return encodeQueryRequest(mt)
+	case *pilosa.QueryResponse:
+		return encodeQueryResponse(mt)
 	}
 	return nil
 }
 
+func encodeQueryRequest(m *pilosa.QueryRequest) *internal.QueryRequest {
+	return &internal.QueryRequest{
+		Query:           m.Query,
+		Shards:          m.Shards,
+		ColumnAttrs:     m.ColumnAttrs,
+		Remote:          m.Remote,
+		ExcludeRowAttrs: m.ExcludeRowAttrs,
+		ExcludeColumns:  m.ExcludeColumns,
+	}
+}
+
+func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse {
+	pb := &internal.QueryResponse{
+		Results:        make([]*internal.QueryResult, len(m.Results)),
+		ColumnAttrSets: EncodeColumnAttrSets(m.ColumnAttrSets),
+	}
+
+	for i := range m.Results {
+		pb.Results[i] = &internal.QueryResult{}
+
+		switch result := m.Results[i].(type) {
+		case *pilosa.Row:
+			pb.Results[i].Type = queryResultTypeRow
+			pb.Results[i].Row = EncodeRow(result)
+		case []pilosa.Pair:
+			pb.Results[i].Type = queryResultTypePairs
+			pb.Results[i].Pairs = EncodePairs(result)
+		case pilosa.ValCount:
+			pb.Results[i].Type = queryResultTypeValCount
+			pb.Results[i].ValCount = EncodeValCount(result)
+		case uint64:
+			pb.Results[i].Type = queryResultTypeUint64
+			pb.Results[i].N = result
+		case bool:
+			pb.Results[i].Type = queryResultTypeBool
+			pb.Results[i].Changed = result
+		case nil:
+			pb.Results[i].Type = queryResultTypeNil
+		}
+	}
+
+	if m.Err != nil {
+		pb.Err = m.Err.Error()
+	}
+
+	return pb
+}
+
 func encodeResizeInstruction(m *pilosa.ResizeInstruction) *internal.ResizeInstruction {
 	return &internal.ResizeInstruction{
 		JobID:         m.JobID,
@@ -602,3 +672,221 @@ func decodeNodeStatus(pb *internal.NodeStatus, m *pilosa.NodeStatus) {
 }
 
 func decodeRecalculateCaches(pb *internal.RecalculateCaches, m *pilosa.RecalculateCaches) {}
+
+func decodeQueryRequest(pb *internal.QueryRequest, m *pilosa.QueryRequest) {
+	m.Query = pb.Query
+	m.Shards = pb.Shards
+	m.ColumnAttrs = pb.ColumnAttrs
+	m.Remote = pb.Remote
+	m.ExcludeRowAttrs = pb.ExcludeRowAttrs
+	m.ExcludeColumns = pb.ExcludeColumns
+}
+
+func decodeQueryResponse(pb *internal.QueryResponse, m *pilosa.QueryResponse) {
+	m.ColumnAttrSets = make([]*pilosa.ColumnAttrSet, len(pb.ColumnAttrSets))
+	decodeColumnAttrSets(pb.ColumnAttrSets, m.ColumnAttrSets)
+	m.Err = errors.New(pb.Err)
+	m.Results = make([]interface{}, len(pb.Results))
+	decodeQueryResults(pb.Results, m.Results)
+
+}
+
+func decodeColumnAttrSets(pb []*internal.ColumnAttrSet, m []*pilosa.ColumnAttrSet) {
+	for i := range pb {
+		decodeColumnAttrSet(pb[i], m[i])
+	}
+}
+
+func decodeColumnAttrSet(pb *internal.ColumnAttrSet, m *pilosa.ColumnAttrSet) {
+	m.ID = pb.ID
+	m.Key = pb.Key
+	m.Attrs = decodeAttrs(pb.Attrs)
+}
+
+func decodeQueryResults(pb []*internal.QueryResult, m []interface{}) {
+	for i := range pb {
+		m[i] = decodeQueryResult(pb[i])
+	}
+}
+
+// QueryResult types.
+const (
+	queryResultTypeNil uint32 = iota
+	queryResultTypeRow
+	queryResultTypePairs
+	queryResultTypeValCount
+	queryResultTypeUint64
+	queryResultTypeBool
+)
+
+func decodeQueryResult(pb *internal.QueryResult) interface{} {
+	switch pb.Type {
+	case queryResultTypeRow:
+		return decodeRow(pb.Row)
+	case queryResultTypePairs:
+		return decodePairs(pb.Pairs)
+	case queryResultTypeValCount:
+		return decodeValCount(pb.ValCount)
+	case queryResultTypeUint64:
+		return pb.N
+	case queryResultTypeBool:
+		return pb.Changed
+	case queryResultTypeNil:
+		return nil
+	}
+	panic(fmt.Sprintf("unknown type: %d", pb.Type))
+}
+
+// DecodeRow converts r from its internal representation.
+func decodeRow(pr *internal.Row) *pilosa.Row {
+	if pr == nil {
+		return nil
+	}
+
+	r := pilosa.NewRow()
+	r.Attrs = decodeAttrs(pr.Attrs)
+	for _, v := range pr.Columns {
+		r.SetBit(v)
+	}
+	return r
+}
+
+func decodeAttrs(pb []*internal.Attr) map[string]interface{} {
+	m := make(map[string]interface{}, len(pb))
+	for i := range pb {
+		key, value := decodeAttr(pb[i])
+		m[key] = value
+	}
+	return m
+}
+
+const (
+	attrTypeString = 1
+	attrTypeInt    = 2
+	attrTypeBool   = 3
+	attrTypeFloat  = 4
+)
+
+func decodeAttr(attr *internal.Attr) (key string, value interface{}) {
+	switch attr.Type {
+	case attrTypeString:
+		return attr.Key, attr.StringValue
+	case attrTypeInt:
+		return attr.Key, attr.IntValue
+	case attrTypeBool:
+		return attr.Key, attr.BoolValue
+	case attrTypeFloat:
+		return attr.Key, attr.FloatValue
+	default:
+		return attr.Key, nil
+	}
+}
+
+func decodePairs(a []*internal.Pair) []pilosa.Pair {
+	other := make([]pilosa.Pair, len(a))
+	for i := range a {
+		other[i] = decodePair(a[i])
+	}
+	return other
+}
+
+func decodePair(pb *internal.Pair) pilosa.Pair {
+	return pilosa.Pair{
+		ID:    pb.ID,
+		Key:   pb.Key,
+		Count: pb.Count,
+	}
+}
+
+func decodeValCount(pb *internal.ValCount) pilosa.ValCount {
+	return pilosa.ValCount{
+		Val:   pb.Val,
+		Count: pb.Count,
+	}
+}
+
+func EncodeColumnAttrSets(a []*pilosa.ColumnAttrSet) []*internal.ColumnAttrSet {
+	other := make([]*internal.ColumnAttrSet, len(a))
+	for i := range a {
+		other[i] = EncodeColumnAttrSet(a[i])
+	}
+	return other
+}
+
+func EncodeColumnAttrSet(set *pilosa.ColumnAttrSet) *internal.ColumnAttrSet {
+	return &internal.ColumnAttrSet{
+		ID:    set.ID,
+		Attrs: encodeAttrs(set.Attrs),
+	}
+}
+
+func EncodeRow(r *pilosa.Row) *internal.Row {
+	if r == nil {
+		return nil
+	}
+
+	return &internal.Row{
+		Columns: r.Columns(),
+		Attrs:   encodeAttrs(r.Attrs),
+	}
+}
+
+func EncodePairs(a pilosa.Pairs) []*internal.Pair {
+	other := make([]*internal.Pair, len(a))
+	for i := range a {
+		other[i] = encodePair(a[i])
+	}
+	return other
+}
+
+func encodePair(p pilosa.Pair) *internal.Pair {
+	return &internal.Pair{
+		ID:    p.ID,
+		Key:   p.Key,
+		Count: p.Count,
+	}
+}
+
+func EncodeValCount(vc pilosa.ValCount) *internal.ValCount {
+	return &internal.ValCount{
+		Val:   vc.Val,
+		Count: vc.Count,
+	}
+}
+
+func encodeAttrs(m map[string]interface{}) []*internal.Attr {
+	keys := make([]string, 0, len(m))
+	for k := range m {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	a := make([]*internal.Attr, len(keys))
+	for i := range keys {
+		a[i] = encodeAttr(keys[i], m[keys[i]])
+	}
+	return a
+}
+
+// encodeAttr converts a key/value pair into an Attr internal representation.
+func encodeAttr(key string, value interface{}) *internal.Attr {
+	pb := &internal.Attr{Key: key}
+	switch value := value.(type) {
+	case string:
+		pb.Type = attrTypeString
+		pb.StringValue = value
+	case float64:
+		pb.Type = attrTypeFloat
+		pb.FloatValue = value
+	case uint64:
+		pb.Type = attrTypeInt
+		pb.IntValue = int64(value)
+	case int64:
+		pb.Type = attrTypeInt
+		pb.IntValue = value
+	case bool:
+		pb.Type = attrTypeBool
+		pb.BoolValue = value
+	}
+	return pb
+}
diff --git a/gossip/gossip.go b/gossip/gossip.go
index 251851077..28b3ac690 100644
--- a/gossip/gossip.go
+++ b/gossip/gossip.go
@@ -28,7 +28,6 @@ import (
 
 	"github.com/hashicorp/memberlist"
 	"github.com/pilosa/pilosa"
-	"github.com/pilosa/pilosa/encoding/proto"
 	"github.com/pilosa/pilosa/toml"
 	"github.com/pkg/errors"
 )
@@ -43,9 +42,8 @@ type GossipMemberSet struct {
 
 	broadcasts *memberlist.TransmitLimitedQueue
 
-	papi       *pilosa.API
-	serializer pilosa.Serializer
-	config     *gossipConfig
+	papi   *pilosa.API
+	config *gossipConfig
 
 	Logger pilosa.Logger
 
@@ -150,9 +148,8 @@ func WithLogger(logger *log.Logger) GossipMemberSetOption {
 func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...GossipMemberSetOption) (*GossipMemberSet, error) {
 	host := api.Node().URI.GetHost()
 	g := &GossipMemberSet{
-		papi:       api,
-		serializer: proto.Serializer{},
-		Logger:     pilosa.NopLogger,
+		papi:   api,
+		Logger: pilosa.NopLogger,
 	}
 
 	// options
@@ -223,7 +220,7 @@ func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...GossipMemberSetO
 
 // NodeMeta implementation of the memberlist.Delegate interface.
 func (g *GossipMemberSet) NodeMeta(limit int) []byte {
-	buf, err := g.serializer.Marshal(g.papi.Node())
+	buf, err := g.papi.Serializer.Marshal(g.papi.Node())
 	if err != nil {
 		g.Logger.Printf("marshal message error: %s", err)
 		return []byte{}
@@ -256,7 +253,7 @@ func (g *GossipMemberSet) LocalState(join bool) []byte {
 	}
 
 	// Marshal nodestate data to bytes.
-	buf, err := pilosa.MarshalInternalMessage(m, g.serializer)
+	buf, err := pilosa.MarshalInternalMessage(m, g.papi.Serializer)
 	if err != nil {
 		g.Logger.Printf("error marshalling nodestate data, err=%s", err)
 		return []byte{}
@@ -279,9 +276,8 @@ func (g *GossipMemberSet) MergeRemoteState(buf []byte, join bool) {
 // Care must be taken that events are processed in a timely manner from
 // the channel, since this delegate will block until an event can be sent.
 type gossipEventReceiver struct {
-	ch         chan memberlist.NodeEvent
-	papi       *pilosa.API
-	serializer pilosa.Serializer
+	ch   chan memberlist.NodeEvent
+	papi *pilosa.API
 
 	logger *log.Logger
 }
@@ -289,10 +285,9 @@ type gossipEventReceiver struct {
 // newGossipEventReceiver returns a new instance of GossipEventReceiver.
 func newGossipEventReceiver(logger *log.Logger, papi *pilosa.API) *gossipEventReceiver {
 	ger := &gossipEventReceiver{
-		ch:         make(chan memberlist.NodeEvent, 1),
-		logger:     logger,
-		papi:       papi,
-		serializer: proto.Serializer{},
+		ch:     make(chan memberlist.NodeEvent, 1),
+		logger: logger,
+		papi:   papi,
 	}
 	go ger.listen()
 	return ger
@@ -327,7 +322,7 @@ func (g *gossipEventReceiver) listen() {
 
 		// Get the node from the event.Node meta data.
 		var n pilosa.Node
-		if err := g.serializer.Unmarshal(e.Node.Meta, &n); err != nil {
+		if err := g.papi.Serializer.Unmarshal(e.Node.Meta, &n); err != nil {
 			panic("failed to unmarshal event node meta into node")
 		}
 
@@ -335,7 +330,7 @@ func (g *gossipEventReceiver) listen() {
 			Event: nodeEventType,
 			Node:  &n,
 		}
-		buf, err := pilosa.MarshalInternalMessage(ne, g.serializer)
+		buf, err := pilosa.MarshalInternalMessage(ne, g.papi.Serializer)
 		if err != nil {
 			panic(err)
 		}
diff --git a/http/handler.go b/http/handler.go
index 9195f05e9..84832b1d4 100644
--- a/http/handler.go
+++ b/http/handler.go
@@ -815,13 +815,12 @@ func (h *Handler) readProtobufQueryRequest(r *http.Request) (*pilosa.QueryReques
 		return nil, errors.Wrap(err, "reading")
 	}
 
-	// Unmarshal into object.
-	var req internal.QueryRequest
-	if err := proto.Unmarshal(body, &req); err != nil {
-		return nil, errors.Wrap(err, "unmarshalling")
+	qreq := &pilosa.QueryRequest{}
+	err = h.API.Serializer.Unmarshal(body, qreq)
+	if err != nil {
+		return nil, errors.Wrap(err, "unmarshalling query request")
 	}
-
-	return decodeQueryRequest(&req), nil
+	return qreq, nil
 }
 
 // readURLQueryRequest parses query parameters from URL parameters from r.
@@ -860,7 +859,7 @@ func (h *Handler) writeQueryResponse(w http.ResponseWriter, r *http.Request, res
 
 // writeProtobufQueryResponse writes the response from the executor to w as protobuf.
 func (h *Handler) writeProtobufQueryResponse(w http.ResponseWriter, resp *pilosa.QueryResponse) error {
-	if buf, err := proto.Marshal(encodeQueryResponse(resp)); err != nil {
+	if buf, err := h.API.Serializer.Marshal(resp); err != nil {
 		return errors.Wrap(err, "marshalling")
 	} else if _, err := w.Write(buf); err != nil {
 		return errors.Wrap(err, "writing")

From e76a90e69b5241d40ea20b56d80e82554428dc91 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Thu, 5 Jul 2018 15:02:33 -0500
Subject: [PATCH 17/32] change Query and QueryNode to use pilosa.* Query
 structs

---
 cache.go                |  8 -------
 client.go               | 14 +++++-------
 encoding/proto/proto.go |  6 ++++-
 executor.go             | 47 ++++++--------------------------------
 executor_test.go        |  2 +-
 fragment.go             |  2 +-
 http/client.go          | 17 +++++++-------
 http/client_test.go     |  9 ++++----
 http/handler.go         | 50 -----------------------------------------
 9 files changed, 32 insertions(+), 123 deletions(-)

diff --git a/cache.go b/cache.go
index 192f38cc7..4c826f5d7 100644
--- a/cache.go
+++ b/cache.go
@@ -409,14 +409,6 @@ func (p Pairs) String() string {
 	return buf.String()
 }
 
-func EncodePairs(a Pairs) []*internal.Pair {
-	other := make([]*internal.Pair, len(a))
-	for i := range a {
-		other[i] = encodePair(a[i])
-	}
-	return other
-}
-
 func decodePairs(a []*internal.Pair) []Pair {
 	other := make([]Pair, len(a))
 	for i := range a {
diff --git a/client.go b/client.go
index 01e0b847a..5c51ae63f 100644
--- a/client.go
+++ b/client.go
@@ -3,8 +3,6 @@ package pilosa
 import (
 	"context"
 	"io"
-
-	"github.com/pilosa/pilosa/internal"
 )
 
 // Bit represents the intersection of a row and a column. It can be specifed by
@@ -35,8 +33,8 @@ type InternalClient interface {
 	Schema(ctx context.Context) ([]*IndexInfo, error)
 	CreateIndex(ctx context.Context, index string, opt IndexOptions) error
 	FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error)
-	Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error)
-	QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error)
+	Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error)
+	QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error)
 	Import(ctx context.Context, index, field string, shard uint64, bits []Bit) error
 	ImportK(ctx context.Context, index, field string, bits []Bit) error
 	EnsureIndex(ctx context.Context, name string, options IndexOptions) error
@@ -55,12 +53,12 @@ type InternalClient interface {
 //===============
 
 type InternalQueryClient interface {
-	QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error)
+	QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error)
 }
 
 type NopInternalQueryClient struct{}
 
-func (n *NopInternalQueryClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) {
+func (n *NopInternalQueryClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
 	return nil, nil
 }
 
@@ -90,10 +88,10 @@ func (n NopInternalClient) CreateIndex(ctx context.Context, index string, opt In
 func (n NopInternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error) {
 	return nil, nil
 }
-func (n NopInternalClient) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) {
+func (n NopInternalClient) Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
 	return nil, nil
 }
-func (n NopInternalClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) {
+func (n NopInternalClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
 	return nil, nil
 }
 func (n NopInternalClient) Import(ctx context.Context, index, field string, shard uint64, bits []Bit) error {
diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go
index 0f2574f1b..341ad46f7 100644
--- a/encoding/proto/proto.go
+++ b/encoding/proto/proto.go
@@ -685,7 +685,11 @@ func decodeQueryRequest(pb *internal.QueryRequest, m *pilosa.QueryRequest) {
 func decodeQueryResponse(pb *internal.QueryResponse, m *pilosa.QueryResponse) {
 	m.ColumnAttrSets = make([]*pilosa.ColumnAttrSet, len(pb.ColumnAttrSets))
 	decodeColumnAttrSets(pb.ColumnAttrSets, m.ColumnAttrSets)
-	m.Err = errors.New(pb.Err)
+	if pb.Err == "" {
+		m.Err = nil
+	} else {
+		m.Err = errors.New(pb.Err)
+	}
 	m.Results = make([]interface{}, len(pb.Results))
 	decodeQueryResults(pb.Results, m.Results)
 
diff --git a/executor.go b/executor.go
index 1a81e92bd..aa2c2fb88 100644
--- a/executor.go
+++ b/executor.go
@@ -337,7 +337,7 @@ func (e *executor) executeBitmapCall(ctx context.Context, index string, c *pql.C
 
 	other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "map reduce")
 	}
 
 	// Attach attributes for Row() calls.
@@ -1392,7 +1392,7 @@ func (e *executor) executeSetColumnAttrs(ctx context.Context, index string, c *p
 // exec executes a PQL query remotely for a set of shards on a node.
 func (e *executor) remoteExec(ctx context.Context, node *Node, index string, q *pql.Query, shards []uint64, opt *execOptions) (results []interface{}, err error) {
 	// Encode request object.
-	pbreq := &internal.QueryRequest{
+	pbreq := &QueryRequest{
 		Query:  q.String(),
 		Shards: shards,
 		Remote: true,
@@ -1403,40 +1403,7 @@ func (e *executor) remoteExec(ctx context.Context, node *Node, index string, q *
 		return nil, err
 	}
 
-	// Return an error, if specified on response.
-	if err := decodeError(pb.Err); err != nil {
-		return nil, err
-	}
-
-	// Return appropriate data for the query.
-	results = make([]interface{}, len(q.Calls))
-	for i, call := range q.Calls {
-		var v interface{}
-		var err error
-
-		switch call.Name {
-		case "Average", "Sum":
-			v, err = decodeValCount(pb.Results[i].GetValCount()), nil
-		case "TopN":
-			v, err = decodePairs(pb.Results[i].GetPairs()), nil
-		case "Count":
-			v, err = pb.Results[i].N, nil
-		case "Set":
-			v, err = pb.Results[i].Changed, nil
-		case "Clear":
-			v, err = pb.Results[i].Changed, nil
-		case "SetRowAttrs":
-		case "SetColumnAttrs":
-		default:
-			v, err = DecodeRow(pb.Results[i].GetRow()), nil
-		}
-		if err != nil {
-			return nil, err
-		}
-
-		results[i] = v
-	}
-	return results, nil
+	return pb.Results, pb.Err
 }
 
 // shardsByNode returns a mapping of nodes to shards.
@@ -1490,7 +1457,7 @@ func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64,
 	for {
 		select {
 		case <-ctx.Done():
-			return nil, ctx.Err()
+			return nil, errors.Wrap(ctx.Err(), "context done")
 		case resp := <-ch:
 			// On error retry against remaining nodes. If an error returns then
 			// the context will cancel and cause all open goroutines to return.
@@ -1500,10 +1467,10 @@ func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64,
 				nodes = Nodes(nodes).Filter(resp.node)
 
 				// Begin mapper against secondary nodes.
-				if err := e.mapper(ctx, ch, nodes, index, resp.shards, c, opt, mapFn, reduceFn); err == errShardUnavailable {
+				if err := e.mapper(ctx, ch, nodes, index, resp.shards, c, opt, mapFn, reduceFn); errors.Cause(err) == errShardUnavailable {
 					return nil, resp.err
 				} else if err != nil {
-					return nil, err
+					return nil, errors.Wrap(err, "calling mapper")
 				}
 				continue
 			}
@@ -1524,7 +1491,7 @@ func (e *executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod
 	// Group shards together by nodes.
 	m, err := e.shardsByNode(nodes, index, shards)
 	if err != nil {
-		return err
+		return errors.Wrap(err, "shards by node")
 	}
 
 	// Execute each node in a separate goroutine.
diff --git a/executor_test.go b/executor_test.go
index f54fb7e33..fc72b00f3 100644
--- a/executor_test.go
+++ b/executor_test.go
@@ -385,7 +385,7 @@ func TestExecutor_Execute_OldPQL(t *testing.T) {
 	hldr.SetBit("i", "f", 1, 0)
 
 	if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetBit(frame=f, row=11, col=1)`}); err == nil || errors.Cause(err).Error() != "unknown call: SetBit" {
-		t.Fatalf("Expected error: 'unknown call: SetBit', got: %v", errors.Cause(err))
+		t.Fatalf("Expected error: 'unknown call: SetBit', got: %v. Full: %v", errors.Cause(err), err)
 	}
 }
 
diff --git a/fragment.go b/fragment.go
index b502e1597..1ae3c5bb3 100644
--- a/fragment.go
+++ b/fragment.go
@@ -1889,7 +1889,7 @@ func (s *fragmentSyncer) syncBlock(id int) error {
 			}
 
 			// Execute query.
-			queryRequest := &internal.QueryRequest{
+			queryRequest := &QueryRequest{
 				Query:  buffers[k].String(),
 				Remote: true,
 			}
diff --git a/http/client.go b/http/client.go
index 41602b778..d8fb7f7ef 100644
--- a/http/client.go
+++ b/http/client.go
@@ -220,22 +220,21 @@ func (c *InternalClient) FragmentNodes(ctx context.Context, index string, shard
 }
 
 // Query executes query against the index.
-func (c *InternalClient) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) {
+func (c *InternalClient) Query(ctx context.Context, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) {
 	return c.QueryNode(ctx, c.defaultURI, index, queryRequest)
 }
 
 // QueryNode executes query against the index, sending the request to the node specified.
-func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) {
+func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) {
 	if index == "" {
 		return nil, pilosa.ErrIndexRequired
 	} else if queryRequest.Query == "" {
 		return nil, pilosa.ErrQueryRequired
 	}
 
-	// Encode request object.
-	buf, err := proto.Marshal(queryRequest)
+	buf, err := c.serializer.Marshal(queryRequest)
 	if err != nil {
-		return nil, errors.Wrap(err, "marshaling")
+		return nil, errors.Wrap(err, "marshaling queryRequest")
 	}
 
 	// Create HTTP request.
@@ -265,11 +264,11 @@ func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index s
 		return nil, errors.New(string(body))
 	}
 
-	qresp := &internal.QueryResponse{}
-	if err := proto.Unmarshal(body, qresp); err != nil {
+	qresp := &pilosa.QueryResponse{}
+	if err := c.serializer.Unmarshal(body, qresp); err != nil {
 		return nil, fmt.Errorf("unmarshal response: %s", err)
-	} else if s := qresp.Err; s != "" {
-		return nil, errors.New(s)
+	} else if qresp.Err != nil {
+		return nil, qresp.Err
 	}
 
 	return qresp, nil
diff --git a/http/client_test.go b/http/client_test.go
index fb1105a27..38a677673 100644
--- a/http/client_test.go
+++ b/http/client_test.go
@@ -24,7 +24,6 @@ import (
 	"github.com/davecgh/go-spew/spew"
 	"github.com/pilosa/pilosa"
 	"github.com/pilosa/pilosa/http"
-	"github.com/pilosa/pilosa/internal"
 	"github.com/pilosa/pilosa/pql"
 	"github.com/pilosa/pilosa/server"
 	"github.com/pilosa/pilosa/test"
@@ -131,7 +130,7 @@ func TestClient_MultiNode(t *testing.T) {
 	client[2] = MustNewClient(c[2].URL(), defaultClient)
 
 	topN := 4
-	queryRequest := &internal.QueryRequest{
+	queryRequest := &pilosa.QueryRequest{
 		Query:  fmt.Sprintf(`TopN(f, n=%d)`, topN),
 		Remote: false,
 	}
@@ -147,17 +146,17 @@ func TestClient_MultiNode(t *testing.T) {
 	}
 
 	// Test must return exactly N results.
-	if len(result.Results[0].Pairs) != topN {
+	if len(result.Results[0].([]pilosa.Pair)) != topN {
 		t.Fatalf("unexpected number of TopN results: %s", spew.Sdump(result))
 	}
-	p := []*internal.Pair{
+	p := []pilosa.Pair{
 		{ID: 100, Count: 12},
 		{ID: 22, Count: 10},
 		{ID: 98, Count: 8},
 		{ID: 99, Count: 7}}
 
 	// Valdidate the Top 4 result counts.
-	if !reflect.DeepEqual(result.Results[0].Pairs, p) {
+	if !reflect.DeepEqual(result.Results[0].([]pilosa.Pair), p) {
 		t.Fatalf("Invalid TopN result set: %s", spew.Sdump(result))
 	}
 
diff --git a/http/handler.go b/http/handler.go
index 84832b1d4..33085879c 100644
--- a/http/handler.go
+++ b/http/handler.go
@@ -1102,56 +1102,6 @@ const (
 	QueryResultTypeBool
 )
 
-func decodeQueryRequest(pb *internal.QueryRequest) *pilosa.QueryRequest {
-	req := &pilosa.QueryRequest{
-		Query:           pb.Query,
-		Shards:          pb.Shards,
-		ColumnAttrs:     pb.ColumnAttrs,
-		Remote:          pb.Remote,
-		ExcludeRowAttrs: pb.ExcludeRowAttrs,
-		ExcludeColumns:  pb.ExcludeColumns,
-	}
-
-	return req
-}
-
-func encodeQueryResponse(resp *pilosa.QueryResponse) *internal.QueryResponse {
-	pb := &internal.QueryResponse{
-		Results:        make([]*internal.QueryResult, len(resp.Results)),
-		ColumnAttrSets: pilosa.EncodeColumnAttrSets(resp.ColumnAttrSets),
-	}
-
-	for i := range resp.Results {
-		pb.Results[i] = &internal.QueryResult{}
-
-		switch result := resp.Results[i].(type) {
-		case *pilosa.Row:
-			pb.Results[i].Type = QueryResultTypeRow
-			pb.Results[i].Row = pilosa.EncodeRow(result)
-		case []pilosa.Pair:
-			pb.Results[i].Type = QueryResultTypePairs
-			pb.Results[i].Pairs = pilosa.EncodePairs(result)
-		case pilosa.ValCount:
-			pb.Results[i].Type = QueryResultTypeValCount
-			pb.Results[i].ValCount = pilosa.EncodeValCount(result)
-		case uint64:
-			pb.Results[i].Type = QueryResultTypeUint64
-			pb.Results[i].N = result
-		case bool:
-			pb.Results[i].Type = QueryResultTypeBool
-			pb.Results[i].Changed = result
-		case nil:
-			pb.Results[i].Type = QueryResultTypeNil
-		}
-	}
-
-	if resp.Err != nil {
-		pb.Err = resp.Err.Error()
-	}
-
-	return pb
-}
-
 // parseUint64Slice returns a slice of uint64s from a comma-delimited string.
 func parseUint64Slice(s string) ([]uint64, error) {
 	var a []uint64

From da4cd8482067f7e1c6cc599d3191f1affc3e56b2 Mon Sep 17 00:00:00 2001
From: Cody Soyland <cody@soyland.org>
Date: Thu, 5 Jul 2018 15:31:07 -0500
Subject: [PATCH 18/32] Remove some dead code

---
 http/client.go   | 10 ----------
 http/handler.go  |  4 ----
 pql/ast.go       | 33 ---------------------------------
 server/config.go |  7 -------
 test/holder.go   |  9 ---------
 5 files changed, 63 deletions(-)

diff --git a/http/client.go b/http/client.go
index 27597a5cb..7c88bd8d3 100644
--- a/http/client.go
+++ b/http/client.go
@@ -27,19 +27,12 @@ import (
 	"sort"
 	"strconv"
 
-	"crypto/tls"
-
 	"github.com/gogo/protobuf/proto"
 	"github.com/pilosa/pilosa"
 	"github.com/pilosa/pilosa/internal"
 	"github.com/pkg/errors"
 )
 
-// ClientOptions represents the configuration for a InternalHTTPClient
-type ClientOptions struct {
-	TLS *tls.Config
-}
-
 // InternalClient represents a client to the Pilosa cluster.
 type InternalClient struct {
 	defaultURI *pilosa.URI
@@ -70,9 +63,6 @@ func NewInternalClientFromURI(defaultURI *pilosa.URI, remoteClient *http.Client)
 	}
 }
 
-// Host returns the host the client was initialized with.
-func (c *InternalClient) Host() *pilosa.URI { return c.defaultURI }
-
 // MaxShardByIndex returns the number of shards on a server by index.
 func (c *InternalClient) MaxShardByIndex(ctx context.Context) (map[string]uint64, error) {
 	return c.maxShardByIndex(ctx)
diff --git a/http/handler.go b/http/handler.go
index 8b2015094..da9a84d9f 100644
--- a/http/handler.go
+++ b/http/handler.go
@@ -1329,10 +1329,6 @@ func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Reques
 	}
 }
 
-func (h *Handler) GetAPI() *pilosa.API {
-	return h.API
-}
-
 type defaultClusterMessageResponse struct{}
 
 func (h *Handler) handleGetTranslateData(w http.ResponseWriter, r *http.Request) {
diff --git a/pql/ast.go b/pql/ast.go
index 0bcc582d4..344292ed4 100644
--- a/pql/ast.go
+++ b/pql/ast.go
@@ -220,23 +220,6 @@ func (q *Query) WriteCallN() int {
 	return n
 }
 
-// HasKeys returns true if any call in the query uses keys and requires translation to ids.
-func (q *Query) HasKeys() bool {
-	for _, call := range q.Calls {
-		if call.Args["col"] != nil {
-			if _, ok := call.Args["col"].(string); ok {
-				return true
-			}
-		}
-		if call.Args["row"] != nil {
-			if _, ok := call.Args["row"].(string); ok {
-				return true
-			}
-		}
-	}
-	return false
-}
-
 // String returns a string representation of the query.
 func (q *Query) String() string {
 	a := make([]string, len(q.Calls))
@@ -309,22 +292,6 @@ func (c *Call) UintSliceArg(key string) ([]uint64, bool, error) {
 	}
 }
 
-// StringArg is for reading the value at key from call.Args as a string. If the
-// key is not in Call.Args, the value of the returned bool will be false, and
-// the error will be nil. An error is returned if the value is not a string.
-func (c *Call) StringArg(key string) (string, bool, error) {
-	val, ok := c.Args[key]
-	if !ok {
-		return "", false, nil
-	}
-	switch tval := val.(type) {
-	case string:
-		return tval, true, nil
-	default:
-		return "", true, fmt.Errorf("could not convert %v of type %T to string in Call.StringArg", tval, tval)
-	}
-}
-
 // Keys returns a list of argument keys in sorted order.
 func (c *Call) Keys() []string {
 	a := make([]string, 0, len(c.Args))
diff --git a/server/config.go b/server/config.go
index 55da45768..73f0b289e 100644
--- a/server/config.go
+++ b/server/config.go
@@ -21,13 +21,6 @@ import (
 	"github.com/pilosa/pilosa/toml"
 )
 
-// Cluster types.
-const (
-	ClusterNone   = ""
-	ClusterStatic = "static"
-	ClusterGossip = "gossip"
-)
-
 // TLSConfig contains TLS configuration
 type TLSConfig struct {
 	// CertificatePath contains the path to the certificate (.crt or .pem file)
diff --git a/test/holder.go b/test/holder.go
index 9cc96fe14..ff87ae02c 100644
--- a/test/holder.go
+++ b/test/holder.go
@@ -81,15 +81,6 @@ func (h *Holder) MustCreateIndexIfNotExists(index string, opt pilosa.IndexOption
 	return &Index{Index: idx}
 }
 
-// MustCreateFieldIfNotExists returns a given field. Panic on error.
-func (h *Holder) MustCreateFieldIfNotExists(index, field string) *Field {
-	f, err := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}).CreateFieldIfNotExists(field, pilosa.OptFieldTypeDefault())
-	if err != nil {
-		panic(err)
-	}
-	return f
-}
-
 // Row returns a Row for a given field.
 func (h *Holder) Row(index, field string, rowID uint64) *pilosa.Row {
 	idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})

From bbb93abd57cd1f0a7195d504f5257c391e247b78 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Thu, 5 Jul 2018 15:35:32 -0500
Subject: [PATCH 19/32] remove lots of unused code

---
 api.go                  |   4 +-
 broadcast.go            |  87 ---------
 cluster.go              | 410 ----------------------------------------
 encoding/proto/proto.go |  10 +-
 executor.go             |  15 --
 field.go                |  34 ----
 holder.go               |  15 --
 index.go                |  24 ---
 row.go                  |  27 ---
 uri.go                  |  29 ---
 10 files changed, 11 insertions(+), 644 deletions(-)

diff --git a/api.go b/api.go
index a9bd02f2d..1abe4502c 100644
--- a/api.go
+++ b/api.go
@@ -770,8 +770,8 @@ func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode
 	// Send the set-coordinator message to new node.
 	err = api.server.SendTo(
 		newNode,
-		&internal.SetCoordinatorMessage{
-			New: EncodeNode(newNode),
+		&SetCoordinatorMessage{
+			New: newNode,
 		})
 	if err != nil {
 		return nil, nil, fmt.Errorf("problem sending SetCoordinator message: %s", err)
diff --git a/broadcast.go b/broadcast.go
index 00835db64..a3ea01a4f 100644
--- a/broadcast.go
+++ b/broadcast.go
@@ -17,8 +17,6 @@ package pilosa
 import (
 	"fmt"
 
-	"github.com/gogo/protobuf/proto"
-	"github.com/pilosa/pilosa/internal"
 	"github.com/pkg/errors"
 )
 
@@ -165,88 +163,3 @@ func getMessageType(m Message) byte {
 		panic(fmt.Sprintf("don't have type for message %#v", m))
 	}
 }
-
-// UnmarshalMessage decodes the byte slice into a protobuf message.
-func UnmarshalMessage(buf []byte) (proto.Message, error) {
-	typ, buf := buf[0], buf[1:]
-	var m proto.Message
-	switch typ {
-	case messageTypeCreateShard:
-		m = &internal.CreateShardMessage{}
-	case messageTypeCreateIndex:
-		m = &internal.CreateIndexMessage{}
-	case messageTypeDeleteIndex:
-		m = &internal.DeleteIndexMessage{}
-	case messageTypeCreateField:
-		m = &internal.CreateFieldMessage{}
-	case messageTypeDeleteField:
-		m = &internal.DeleteFieldMessage{}
-	case messageTypeCreateView:
-		m = &internal.CreateViewMessage{}
-	case messageTypeDeleteView:
-		m = &internal.DeleteViewMessage{}
-	case messageTypeClusterStatus:
-		m = &internal.ClusterStatus{}
-	case messageTypeResizeInstruction:
-		m = &internal.ResizeInstruction{}
-	case messageTypeResizeInstructionComplete:
-		m = &internal.ResizeInstructionComplete{}
-	case messageTypeSetCoordinator:
-		m = &internal.SetCoordinatorMessage{}
-	case messageTypeUpdateCoordinator:
-		m = &internal.UpdateCoordinatorMessage{}
-	case messageTypeNodeState:
-		m = &internal.NodeStateMessage{}
-	case messageTypeRecalculateCaches:
-		m = &internal.RecalculateCaches{}
-	case messageTypeNodeEvent:
-		m = &internal.NodeEventMessage{}
-	case messageTypeNodeStatus:
-		m = &internal.NodeStatus{}
-	default:
-		return nil, fmt.Errorf("invalid message type: %d", typ)
-	}
-
-	if err := proto.Unmarshal(buf, m); err != nil {
-		return nil, errors.Wrap(err, "unmarshalling")
-	}
-	return m, nil
-}
-
-func decode(m proto.Message) Message {
-	switch mt := m.(type) {
-	case *internal.CreateShardMessage:
-		return decodeCreateShardMessage(mt)
-	case *internal.CreateIndexMessage:
-		return decodeCreateIndexMessage(mt)
-	case *internal.DeleteIndexMessage:
-		return decodeDeleteIndexMessage(mt)
-	case *internal.CreateFieldMessage:
-		return decodeCreateFieldMessage(mt)
-	case *internal.DeleteFieldMessage:
-		return decodeDeleteFieldMessage(mt)
-	case *internal.CreateViewMessage:
-		return decodeCreateViewMessage(mt)
-	case *internal.DeleteViewMessage:
-		return decodeDeleteViewMessage(mt)
-	case *internal.ClusterStatus:
-		return decodeClusterStatus(mt)
-	case *internal.ResizeInstruction:
-		return decodeResizeInstruction(mt)
-	case *internal.ResizeInstructionComplete:
-		return decodeResizeInstructionComplete(mt)
-	case *internal.SetCoordinatorMessage:
-		return decodeSetCoordinatorMessage(mt)
-	case *internal.UpdateCoordinatorMessage:
-		return decodeUpdateCoordinatorMessage(mt)
-	case *internal.NodeStateMessage:
-		return decodeNodeStateMessage(mt)
-	case *internal.RecalculateCaches:
-		return decodeRecalculateCaches(mt)
-	case *internal.NodeEventMessage:
-		return decodeNodeEventMessage(mt)
-	case *internal.NodeStatus:
-		return decodeNodeStatus(mt)
-	}
-	return nil
-}
diff --git a/cluster.go b/cluster.go
index e2f29fde5..541c83642 100644
--- a/cluster.go
+++ b/cluster.go
@@ -1757,28 +1757,6 @@ type ResizeInstruction struct {
 	ClusterStatus *ClusterStatus
 }
 
-func decodeResizeInstruction(ri *internal.ResizeInstruction) *ResizeInstruction {
-	return &ResizeInstruction{
-		JobID:         ri.JobID,
-		Node:          DecodeNode(ri.Node),
-		Coordinator:   DecodeNode(ri.Coordinator),
-		Sources:       decodeResizeSources(ri.Sources),
-		Schema:        decodeSchema(ri.Schema),
-		ClusterStatus: decodeClusterStatus(ri.ClusterStatus),
-	}
-}
-
-func encodeResizeInstruction(m *ResizeInstruction) *internal.ResizeInstruction {
-	return &internal.ResizeInstruction{
-		JobID:         m.JobID,
-		Node:          EncodeNode(m.Node),
-		Coordinator:   EncodeNode(m.Coordinator),
-		Sources:       encodeResizeSources(m.Sources),
-		Schema:        encodeSchema(m.Schema),
-		ClusterStatus: encodeClusterStatus(m.ClusterStatus),
-	}
-}
-
 type ResizeSource struct {
 	Node  *Node  `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"`
 	Index string `protobuf:"bytes,2,opt,name=Index,proto3" json:"Index,omitempty"`
@@ -1787,192 +1765,11 @@ type ResizeSource struct {
 	Shard uint64 `protobuf:"varint,5,opt,name=Shard,proto3" json:"Shard,omitempty"`
 }
 
-func decodeResizeSources(srcs []*internal.ResizeSource) []*ResizeSource {
-	new := make([]*ResizeSource, 0, len(srcs))
-	for _, src := range srcs {
-		new = append(new, decodeResizeSource(src))
-	}
-	return new
-}
-
-func encodeResizeSources(srcs []*ResizeSource) []*internal.ResizeSource {
-	new := make([]*internal.ResizeSource, 0, len(srcs))
-	for _, src := range srcs {
-		new = append(new, encodeResizeSource(src))
-	}
-	return new
-}
-
-func decodeResizeSource(rs *internal.ResizeSource) *ResizeSource {
-	return &ResizeSource{
-		Node:  DecodeNode(rs.Node),
-		Index: rs.Index,
-		Field: rs.Field,
-		View:  rs.View,
-		Shard: rs.Shard,
-	}
-}
-
-func encodeResizeSource(m *ResizeSource) *internal.ResizeSource {
-	return &internal.ResizeSource{
-		Node:  EncodeNode(m.Node),
-		Index: m.Index,
-		Field: m.Field,
-		View:  m.View,
-		Shard: m.Shard,
-	}
-}
-
 // Schema is a schema
 type Schema struct {
 	Indexes []*IndexInfo
 }
 
-func decodeSchema(s *internal.Schema) *Schema {
-	return &Schema{
-		Indexes: decodeIndexes(s.Indexes),
-	}
-}
-
-func encodeSchema(m *Schema) *internal.Schema {
-	return &internal.Schema{
-		Indexes: encodeIndexInfos(m.Indexes),
-	}
-}
-
-func decodeIndexes(idxs []*internal.Index) []*IndexInfo {
-	new := make([]*IndexInfo, 0, len(idxs))
-	for _, idx := range idxs {
-		new = append(new, decodeIndex(idx))
-	}
-	return new
-}
-
-func encodeIndexInfos(idxs []*IndexInfo) []*internal.Index {
-	new := make([]*internal.Index, 0, len(idxs))
-	for _, idx := range idxs {
-		new = append(new, encodeIndexInfo(idx))
-	}
-	return new
-}
-
-func decodeIndex(idx *internal.Index) *IndexInfo {
-	return &IndexInfo{
-		Name:   idx.Name,
-		Fields: decodeFields(idx.Fields),
-	}
-}
-
-func encodeIndexInfo(idx *IndexInfo) *internal.Index {
-	return &internal.Index{
-		Name:   idx.Name,
-		Fields: encodeFieldInfos(idx.Fields),
-	}
-}
-
-func decodeFields(fs []*internal.Field) []*FieldInfo {
-	new := make([]*FieldInfo, 0, len(fs))
-	for _, f := range fs {
-		new = append(new, decodeField(f))
-	}
-	return new
-}
-
-func encodeFieldInfos(fs []*FieldInfo) []*internal.Field {
-	new := make([]*internal.Field, 0, len(fs))
-	for _, f := range fs {
-		new = append(new, encodeFieldInfo(f))
-	}
-	return new
-}
-
-func decodeField(f *internal.Field) *FieldInfo {
-	fi := &FieldInfo{
-		Name:    f.Name,
-		Options: *decodeFieldOptions(f.Meta),
-		Views:   make([]*ViewInfo, 0, len(f.Views)),
-	}
-	for _, viewname := range f.Views {
-		fi.Views = append(fi.Views, &ViewInfo{Name: viewname})
-	}
-	return fi
-}
-
-func encodeFieldInfo(f *FieldInfo) *internal.Field {
-	ifield := &internal.Field{
-		Name:  f.Name,
-		Meta:  encodeFieldOptions(&f.Options),
-		Views: make([]string, 0, len(f.Views)),
-	}
-
-	for _, viewinfo := range f.Views {
-		ifield.Views = append(ifield.Views, viewinfo.Name)
-	}
-	return ifield
-}
-
-// EncodeNodes converts a slice of Nodes into its internal representation.
-func EncodeNodes(a []*Node) []*internal.Node {
-	other := make([]*internal.Node, len(a))
-	for i := range a {
-		other[i] = EncodeNode(a[i])
-	}
-	return other
-}
-
-// EncodeNode converts a Node into its internal representation.
-func EncodeNode(n *Node) *internal.Node {
-	return &internal.Node{
-		ID:            n.ID,
-		URI:           n.URI.Encode(),
-		IsCoordinator: n.IsCoordinator,
-	}
-}
-
-// DecodeNodes converts a proto message into a slice of Nodes.
-func DecodeNodes(a []*internal.Node) []*Node {
-	if len(a) == 0 {
-		return nil
-	}
-	other := make([]*Node, len(a))
-	for i := range a {
-		other[i] = DecodeNode(a[i])
-	}
-	return other
-}
-
-func decodeClusterStatus(cs *internal.ClusterStatus) *ClusterStatus {
-	return &ClusterStatus{
-		State:     cs.State,
-		ClusterID: cs.ClusterID,
-		Nodes:     DecodeNodes(cs.Nodes),
-	}
-}
-
-func encodeClusterStatus(m *ClusterStatus) *internal.ClusterStatus {
-	return &internal.ClusterStatus{
-		State:     m.State,
-		ClusterID: m.ClusterID,
-		Nodes:     EncodeNodes(m.Nodes),
-	}
-}
-
-// DecodeNode converts a proto message into a Node.
-func DecodeNode(node *internal.Node) *Node {
-	return &Node{
-		ID:            node.ID,
-		URI:           decodeURI(node.URI),
-		IsCoordinator: node.IsCoordinator,
-	}
-}
-
-func DecodeNodeEvent(ne *internal.NodeEventMessage) *NodeEvent {
-	return &NodeEvent{
-		Event: NodeEventType(ne.Event),
-		Node:  DecodeNode(ne.Node),
-	}
-}
-
 func encodeTopology(topology *Topology) *internal.Topology {
 	if topology == nil {
 		return nil
@@ -2004,267 +1801,60 @@ type CreateShardMessage struct {
 	Shard uint64
 }
 
-func encodeCreateShardMessage(m *CreateShardMessage) *internal.CreateShardMessage {
-	return &internal.CreateShardMessage{
-		Index: m.Index,
-		Shard: m.Shard,
-	}
-}
-
-func decodeCreateShardMessage(pb *internal.CreateShardMessage) *CreateShardMessage {
-	return &CreateShardMessage{
-		Index: pb.Index,
-		Shard: pb.Shard,
-	}
-}
-
 type CreateIndexMessage struct {
 	Index string
 	Meta  *IndexOptions
 }
 
-func encodeCreateIndexMessage(m *CreateIndexMessage) *internal.CreateIndexMessage {
-	return &internal.CreateIndexMessage{
-		Index: m.Index,
-		Meta:  encodeIndexMeta(m.Meta),
-	}
-}
-
-func decodeCreateIndexMessage(pb *internal.CreateIndexMessage) *CreateIndexMessage {
-	return &CreateIndexMessage{
-		Index: pb.Index,
-		Meta:  decodeIndexMeta(pb.Meta),
-	}
-}
-
-func encodeIndexMeta(m *IndexOptions) *internal.IndexMeta {
-	return &internal.IndexMeta{
-		Keys: m.Keys,
-	}
-}
-
-func decodeIndexMeta(pb *internal.IndexMeta) *IndexOptions {
-	return &IndexOptions{
-		Keys: pb.Keys,
-	}
-}
-
 type DeleteIndexMessage struct {
 	Index string
 }
 
-func encodeDeleteIndexMessage(m *DeleteIndexMessage) *internal.DeleteIndexMessage {
-	return &internal.DeleteIndexMessage{
-		Index: m.Index,
-	}
-}
-
-func decodeDeleteIndexMessage(pb *internal.DeleteIndexMessage) *DeleteIndexMessage {
-	return &DeleteIndexMessage{
-		Index: pb.Index,
-	}
-}
-
 type CreateFieldMessage struct {
 	Index string
 	Field string
 	Meta  *FieldOptions
 }
 
-func encodeCreateFieldMessage(m *CreateFieldMessage) *internal.CreateFieldMessage {
-	return &internal.CreateFieldMessage{
-		Index: m.Index,
-		Field: m.Field,
-		Meta:  encodeFieldOptions(m.Meta),
-	}
-}
-
-func decodeCreateFieldMessage(pb *internal.CreateFieldMessage) *CreateFieldMessage {
-	return &CreateFieldMessage{
-		Index: pb.Index,
-		Field: pb.Field,
-		Meta:  decodeFieldOptions(pb.Meta),
-	}
-}
-
 type DeleteFieldMessage struct {
 	Index string
 	Field string
 }
 
-func encodeDeleteFieldMessage(m *DeleteFieldMessage) *internal.DeleteFieldMessage {
-	return &internal.DeleteFieldMessage{
-		Index: m.Index,
-		Field: m.Field,
-	}
-}
-
-func decodeDeleteFieldMessage(pb *internal.DeleteFieldMessage) *DeleteFieldMessage {
-	return &DeleteFieldMessage{
-		Index: pb.Index,
-		Field: pb.Field,
-	}
-}
-
 type CreateViewMessage struct {
 	Index string
 	Field string
 	View  string
 }
-
-func encodeCreateViewMessage(m *CreateViewMessage) *internal.CreateViewMessage {
-	return &internal.CreateViewMessage{
-		Index: m.Index,
-		Field: m.Field,
-		View:  m.View,
-	}
-}
-
-func decodeCreateViewMessage(pb *internal.CreateViewMessage) *CreateViewMessage {
-	return &CreateViewMessage{
-		Index: pb.Index,
-		Field: pb.Field,
-		View:  pb.View,
-	}
-}
-
 type DeleteViewMessage struct {
 	Index string
 	Field string
 	View  string
 }
 
-func encodeDeleteViewMessage(m *DeleteViewMessage) *internal.DeleteViewMessage {
-	return &internal.DeleteViewMessage{
-		Index: m.Index,
-		Field: m.Field,
-		View:  m.View,
-	}
-}
-
-func decodeDeleteViewMessage(pb *internal.DeleteViewMessage) *DeleteViewMessage {
-	return &DeleteViewMessage{
-		Index: pb.Index,
-		Field: pb.Field,
-		View:  pb.View,
-	}
-}
-
 type ResizeInstructionComplete struct {
 	JobID int64
 	Node  *Node
 	Error string
 }
 
-func encodeResizeInstructionComplete(m *ResizeInstructionComplete) *internal.ResizeInstructionComplete {
-	return &internal.ResizeInstructionComplete{
-		JobID: m.JobID,
-		Node:  EncodeNode(m.Node),
-		Error: m.Error,
-	}
-}
-
-func decodeResizeInstructionComplete(pb *internal.ResizeInstructionComplete) *ResizeInstructionComplete {
-	return &ResizeInstructionComplete{
-		JobID: pb.JobID,
-		Node:  DecodeNode(pb.Node),
-		Error: pb.Error,
-	}
-}
-
 type SetCoordinatorMessage struct {
 	New *Node
 }
 
-func encodeSetCoordinatorMessage(m *SetCoordinatorMessage) *internal.SetCoordinatorMessage {
-	return &internal.SetCoordinatorMessage{
-		New: EncodeNode(m.New),
-	}
-}
-
-func decodeSetCoordinatorMessage(pb *internal.SetCoordinatorMessage) *SetCoordinatorMessage {
-	return &SetCoordinatorMessage{
-		New: DecodeNode(pb.New),
-	}
-}
-
 type UpdateCoordinatorMessage struct {
 	New *Node
 }
 
-func encodeUpdateCoordinatorMessage(m *UpdateCoordinatorMessage) *internal.UpdateCoordinatorMessage {
-	return &internal.UpdateCoordinatorMessage{
-		New: EncodeNode(m.New),
-	}
-}
-
-func decodeUpdateCoordinatorMessage(pb *internal.UpdateCoordinatorMessage) *UpdateCoordinatorMessage {
-	return &UpdateCoordinatorMessage{
-		New: DecodeNode(pb.New),
-	}
-}
-
 type NodeStateMessage struct {
 	NodeID string `protobuf:"bytes,1,opt,name=NodeID,proto3" json:"NodeID,omitempty"`
 	State  string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"`
 }
 
-func encodeNodeStateMessage(m *NodeStateMessage) *internal.NodeStateMessage {
-	return &internal.NodeStateMessage{
-		NodeID: m.NodeID,
-		State:  m.State,
-	}
-}
-
-func decodeNodeStateMessage(pb *internal.NodeStateMessage) *NodeStateMessage {
-	return &NodeStateMessage{
-		NodeID: pb.NodeID,
-		State:  pb.State,
-	}
-}
-
-func encodeNodeEventMessage(m *NodeEvent) *internal.NodeEventMessage {
-	return &internal.NodeEventMessage{
-		Event: uint32(m.Event),
-		Node:  EncodeNode(m.Node),
-	}
-}
-
-func decodeNodeEventMessage(pb *internal.NodeEventMessage) *NodeEvent {
-	return &NodeEvent{
-		Event: NodeEventType(pb.Event),
-		Node:  DecodeNode(pb.Node),
-	}
-}
-
 type NodeStatus struct {
 	Node      *Node
 	MaxShards map[string]uint64
 	Schema    *Schema
 }
 
-func encodeNodeStatus(m *NodeStatus) *internal.NodeStatus {
-	return &internal.NodeStatus{
-		Node:      EncodeNode(m.Node),
-		MaxShards: &internal.MaxShards{Standard: m.MaxShards},
-		Schema:    encodeSchema(m.Schema),
-	}
-}
-
-func decodeNodeStatus(pb *internal.NodeStatus) *NodeStatus {
-	return &NodeStatus{
-		Node:      DecodeNode(pb.Node),
-		MaxShards: pb.MaxShards.Standard,
-		Schema:    decodeSchema(pb.Schema),
-	}
-}
-
 type RecalculateCaches struct{}
-
-func decodeRecalculateCaches(pb *internal.RecalculateCaches) *RecalculateCaches {
-	return &RecalculateCaches{}
-}
-
-func encodeRecalculateCaches(*RecalculateCaches) *internal.RecalculateCaches {
-	return &internal.RecalculateCaches{}
-}
diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go
index 341ad46f7..003e9cd94 100644
--- a/encoding/proto/proto.go
+++ b/encoding/proto/proto.go
@@ -375,11 +375,19 @@ func EncodeNodes(a []*pilosa.Node) []*internal.Node {
 func encodeNode(n *pilosa.Node) *internal.Node {
 	return &internal.Node{
 		ID:            n.ID,
-		URI:           n.URI.Encode(),
+		URI:           encodeURI(n.URI),
 		IsCoordinator: n.IsCoordinator,
 	}
 }
 
+func encodeURI(u pilosa.URI) *internal.URI {
+	return &internal.URI{
+		Scheme: u.Scheme,
+		Host:   u.Host,
+		Port:   uint32(u.Port),
+	}
+}
+
 func encodeClusterStatus(m *pilosa.ClusterStatus) *internal.ClusterStatus {
 	return &internal.ClusterStatus{
 		State:     m.State,
diff --git a/executor.go b/executor.go
index aa2c2fb88..399da685d 100644
--- a/executor.go
+++ b/executor.go
@@ -20,7 +20,6 @@ import (
 	"sort"
 	"time"
 
-	"github.com/pilosa/pilosa/internal"
 	"github.com/pilosa/pilosa/pql"
 	"github.com/pkg/errors"
 )
@@ -1738,20 +1737,6 @@ func (vc *ValCount) Add(other ValCount) ValCount {
 	}
 }
 
-func EncodeValCount(vc ValCount) *internal.ValCount {
-	return &internal.ValCount{
-		Val:   vc.Val,
-		Count: vc.Count,
-	}
-}
-
-func decodeValCount(pb *internal.ValCount) ValCount {
-	return ValCount{
-		Val:   pb.Val,
-		Count: pb.Count,
-	}
-}
-
 // Smaller returns the smaller of the two ValCounts.
 func (vc *ValCount) Smaller(other ValCount) ValCount {
 	if vc.Count == 0 || (other.Val < vc.Val && other.Count > 0) {
diff --git a/field.go b/field.go
index fb7e9af4f..0ef5630e3 100644
--- a/field.go
+++ b/field.go
@@ -1088,25 +1088,6 @@ func (f *Field) MarshalJSON() ([]byte, error) {
 	return json.Marshal(thing)
 }
 
-// encodeFields converts a into its internal representation.
-func encodeFields(a []*Field) []*internal.Field {
-	other := make([]*internal.Field, len(a))
-	for i := range a {
-		other[i] = encodeField(a[i])
-	}
-	return other
-}
-
-// encodeField converts f into its internal representation.
-func encodeField(f *Field) *internal.Field {
-	fo := f.options
-	return &internal.Field{
-		Name:  f.name,
-		Meta:  fo.Encode(),
-		Views: f.viewNames(),
-	}
-}
-
 type fieldSlice []*Field
 
 func (p fieldSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
@@ -1170,21 +1151,6 @@ func encodeFieldOptions(o *FieldOptions) *internal.FieldOptions {
 	}
 }
 
-func decodeFieldOptions(options *internal.FieldOptions) *FieldOptions {
-	if options == nil {
-		return nil
-	}
-	return &FieldOptions{
-		Type:        options.Type,
-		CacheType:   options.CacheType,
-		CacheSize:   options.CacheSize,
-		Min:         options.Min,
-		Max:         options.Max,
-		TimeQuantum: TimeQuantum(options.TimeQuantum),
-		Keys:        options.Keys,
-	}
-}
-
 func (o *FieldOptions) MarshalJSON() ([]byte, error) {
 	switch o.Type {
 	case FieldTypeSet:
diff --git a/holder.go b/holder.go
index 10e147098..cb067db92 100644
--- a/holder.go
+++ b/holder.go
@@ -27,7 +27,6 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/pilosa/pilosa/internal"
 	"github.com/pkg/errors"
 	uuid "github.com/satori/go.uuid"
 )
@@ -256,20 +255,6 @@ func (h *Holder) applySchema(schema *Schema) error {
 	return nil
 }
 
-// encodeMaxShards creates and internal representation of max shards.
-func (h *Holder) encodeMaxShards() *internal.MaxShards {
-	return &internal.MaxShards{
-		Standard: h.maxShards(),
-	}
-}
-
-// encodeSchema creates an internal representation of schema.
-func (h *Holder) encodeSchema() *internal.Schema {
-	return &internal.Schema{
-		Indexes: EncodeIndexes(h.Indexes()),
-	}
-}
-
 // IndexPath returns the path where a given index is stored.
 func (h *Holder) IndexPath(name string) string { return filepath.Join(h.Path, name) }
 
diff --git a/index.go b/index.go
index 98f50eced..7451030cd 100644
--- a/index.go
+++ b/index.go
@@ -403,35 +403,11 @@ func (p indexInfoSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
 func (p indexInfoSlice) Len() int           { return len(p) }
 func (p indexInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
 
-// EncodeIndexes converts a into its internal representation.
-func EncodeIndexes(a []*Index) []*internal.Index {
-	other := make([]*internal.Index, len(a))
-	for i := range a {
-		other[i] = encodeIndex(a[i])
-	}
-	return other
-}
-
-// encodeIndex converts d into its internal representation.
-func encodeIndex(d *Index) *internal.Index {
-	return &internal.Index{
-		Name:   d.name,
-		Fields: encodeFields(d.Fields()),
-	}
-}
-
 // IndexOptions represents options to set when initializing an index.
 type IndexOptions struct {
 	Keys bool `json:"keys"`
 }
 
-// Encode converts i into its internal representation.
-func (i *IndexOptions) Encode() *internal.IndexMeta {
-	return &internal.IndexMeta{
-		Keys: i.Keys,
-	}
-}
-
 // hasTime returns true if a contains a non-nil time.
 func hasTime(a []*time.Time) bool {
 	for _, t := range a {
diff --git a/row.go b/row.go
index cbfa6b270..2134026e9 100644
--- a/row.go
+++ b/row.go
@@ -18,7 +18,6 @@ import (
 	"encoding/json"
 	"sort"
 
-	"github.com/pilosa/pilosa/internal"
 	"github.com/pilosa/pilosa/roaring"
 )
 
@@ -271,32 +270,6 @@ func (r *Row) Columns() []uint64 {
 	return a
 }
 
-// EncodeRow converts r into its internal representation.
-func EncodeRow(r *Row) *internal.Row {
-	if r == nil {
-		return nil
-	}
-
-	return &internal.Row{
-		Columns: r.Columns(),
-		Attrs:   encodeAttrs(r.Attrs),
-	}
-}
-
-// DecodeRow converts r from its internal representation.
-func DecodeRow(pr *internal.Row) *Row {
-	if pr == nil {
-		return nil
-	}
-
-	r := NewRow()
-	r.Attrs = decodeAttrs(pr.Attrs)
-	for _, v := range pr.Columns {
-		r.SetBit(v)
-	}
-	return r
-}
-
 // Union performs a union on a slice of rows.
 func Union(rows []*Row) *Row {
 	other := rows[0]
diff --git a/uri.go b/uri.go
index 2058f70fa..b58678381 100644
--- a/uri.go
+++ b/uri.go
@@ -21,7 +21,6 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/pilosa/pilosa/internal"
 	"github.com/pkg/errors"
 )
 
@@ -206,34 +205,6 @@ func parseAddress(address string) (uri *URI, err error) {
 	return uri, nil
 }
 
-// Encode converts o into its internal representation.
-func (u URI) Encode() *internal.URI {
-	return encodeURI(u)
-}
-
-func encodeURI(u URI) *internal.URI {
-	return &internal.URI{
-		Scheme: u.Scheme,
-		Host:   u.Host,
-		Port:   uint32(u.Port),
-	}
-}
-
-func DecodeURI(i *internal.URI) URI {
-	return decodeURI(i)
-}
-
-func decodeURI(i *internal.URI) URI {
-	if i == nil {
-		return URI{}
-	}
-	return URI{
-		Scheme: i.Scheme,
-		Host:   i.Host,
-		Port:   uint16(i.Port),
-	}
-}
-
 // MarshalJSON marshals URI into a JSON-encoded byte slice.
 func (u *URI) MarshalJSON() ([]byte, error) {
 	var output struct {

From db2a53223d591a396c5d50964d3e675069616af9 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Thu, 5 Jul 2018 16:02:18 -0500
Subject: [PATCH 20/32] remove internal references from api and http/*

---
 api.go                  |  14 ++---
 encoding/proto/proto.go | 134 +++++++++++++++++++++++++++++++++++++++-
 handler.go              |  37 +++++++++++
 http/client.go          |  34 +++++-----
 http/handler.go         |  12 ++--
 5 files changed, 197 insertions(+), 34 deletions(-)

diff --git a/api.go b/api.go
index 1abe4502c..181d1b3fb 100644
--- a/api.go
+++ b/api.go
@@ -26,8 +26,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/gogo/protobuf/proto"
-	"github.com/pilosa/pilosa/internal"
 	"github.com/pilosa/pilosa/pql"
 	"github.com/pkg/errors"
 )
@@ -437,8 +435,8 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte,
 	if err != nil {
 		return nil, NewBadRequestError(errors.Wrap(err, "read body error"))
 	}
-	var req internal.BlockDataRequest
-	if err := proto.Unmarshal(reqBytes, &req); err != nil {
+	var req BlockDataRequest
+	if err := api.Serializer.Unmarshal(reqBytes, &req); err != nil {
 		return nil, NewBadRequestError(errors.Wrap(err, "unmarshal body error"))
 	}
 
@@ -448,11 +446,11 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte,
 		return nil, ErrFragmentNotFound
 	}
 
-	var resp = internal.BlockDataResponse{}
+	var resp = BlockDataResponse{}
 	resp.RowIDs, resp.ColumnIDs = f.blockData(int(req.Block))
 
 	// Encode response.
-	buf, err := proto.Marshal(&resp)
+	buf, err := api.Serializer.Marshal(&resp)
 	if err != nil {
 		return nil, errors.Wrap(err, "merge block response encoding error")
 	}
@@ -657,7 +655,7 @@ func (api *API) FieldAttrDiff(ctx context.Context, indexName string, fieldName s
 }
 
 // Import bulk imports data into a particular index,field,shard.
-func (api *API) Import(ctx context.Context, req internal.ImportRequest) error {
+func (api *API) Import(ctx context.Context, req *ImportRequest) error {
 	if err := api.validate(apiImport); err != nil {
 		return errors.Wrap(err, "validating api method")
 	}
@@ -686,7 +684,7 @@ func (api *API) Import(ctx context.Context, req internal.ImportRequest) error {
 }
 
 // ImportValue bulk imports values into a particular field.
-func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest) error {
+func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest) error {
 	if err := api.validate(apiImportValue); err != nil {
 		return errors.Wrap(err, "validating api method")
 	}
diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go
index 003e9cd94..ed751247e 100644
--- a/encoding/proto/proto.go
+++ b/encoding/proto/proto.go
@@ -178,7 +178,46 @@ func (Serializer) Unmarshal(buf []byte, m pilosa.Message) error {
 		}
 		decodeQueryResponse(msg, mt)
 		return nil
-
+	case *pilosa.ImportRequest:
+		msg := &internal.ImportRequest{}
+		err := proto.Unmarshal(buf, msg)
+		if err != nil {
+			return errors.Wrap(err, "unmarshaling ImportRequest")
+		}
+		decodeImportRequest(msg, mt)
+		return nil
+	case *pilosa.ImportValueRequest:
+		msg := &internal.ImportValueRequest{}
+		err := proto.Unmarshal(buf, msg)
+		if err != nil {
+			return errors.Wrap(err, "unmarshaling ImportValueRequest")
+		}
+		decodeImportValueRequest(msg, mt)
+		return nil
+	case *pilosa.ImportResponse:
+		msg := &internal.ImportResponse{}
+		err := proto.Unmarshal(buf, msg)
+		if err != nil {
+			return errors.Wrap(err, "unmarshaling ImportResponse")
+		}
+		decodeImportResponse(msg, mt)
+		return nil
+	case *pilosa.BlockDataRequest:
+		msg := &internal.BlockDataRequest{}
+		err := proto.Unmarshal(buf, msg)
+		if err != nil {
+			return errors.Wrap(err, "unmarshaling BlockDataRequest")
+		}
+		decodeBlockDataRequest(msg, mt)
+		return nil
+	case *pilosa.BlockDataResponse:
+		msg := &internal.BlockDataResponse{}
+		err := proto.Unmarshal(buf, msg)
+		if err != nil {
+			return errors.Wrap(err, "unmarshaling BlockDataResponse")
+		}
+		decodeBlockDataResponse(msg, mt)
+		return nil
 	default:
 		panic(fmt.Sprintf("unhandled pilosa.Message of type %T: %#v", mt, m))
 	}
@@ -224,10 +263,66 @@ func encodeToProto(m pilosa.Message) proto.Message {
 		return encodeQueryRequest(mt)
 	case *pilosa.QueryResponse:
 		return encodeQueryResponse(mt)
+	case *pilosa.ImportRequest:
+		return encodeImportRequest(mt)
+	case *pilosa.ImportValueRequest:
+		return encodeImportValueRequest(mt)
+	case *pilosa.ImportResponse:
+		return encodeImportResponse(mt)
+	case *pilosa.BlockDataRequest:
+		return encodeBlockDataRequest(mt)
+	case *pilosa.BlockDataResponse:
+		return encodeBlockDataResponse(mt)
 	}
 	return nil
 }
 
+func encodeBlockDataRequest(m *pilosa.BlockDataRequest) *internal.BlockDataRequest {
+	return &internal.BlockDataRequest{
+		Index: m.Index,
+		Field: m.Field,
+		View:  m.View,
+		Shard: m.Shard,
+		Block: m.Block,
+	}
+}
+func encodeBlockDataResponse(m *pilosa.BlockDataResponse) *internal.BlockDataResponse {
+	return &internal.BlockDataResponse{
+		RowIDs:    m.RowIDs,
+		ColumnIDs: m.ColumnIDs,
+	}
+}
+
+func encodeImportResponse(m *pilosa.ImportResponse) *internal.ImportResponse {
+	return &internal.ImportResponse{
+		Err: m.Err,
+	}
+}
+
+func encodeImportRequest(m *pilosa.ImportRequest) *internal.ImportRequest {
+	return &internal.ImportRequest{
+		Index:      m.Index,
+		Field:      m.Field,
+		Shard:      m.Shard,
+		RowIDs:     m.RowIDs,
+		ColumnIDs:  m.ColumnIDs,
+		RowKeys:    m.RowKeys,
+		ColumnKeys: m.ColumnKeys,
+		Timestamps: m.Timestamps,
+	}
+}
+
+func encodeImportValueRequest(m *pilosa.ImportValueRequest) *internal.ImportValueRequest {
+	return &internal.ImportValueRequest{
+		Index:      m.Index,
+		Field:      m.Field,
+		Shard:      m.Shard,
+		ColumnIDs:  m.ColumnIDs,
+		ColumnKeys: m.ColumnKeys,
+		Values:     m.Values,
+	}
+}
+
 func encodeQueryRequest(m *pilosa.QueryRequest) *internal.QueryRequest {
 	return &internal.QueryRequest{
 		Query:           m.Query,
@@ -690,6 +785,43 @@ func decodeQueryRequest(pb *internal.QueryRequest, m *pilosa.QueryRequest) {
 	m.ExcludeColumns = pb.ExcludeColumns
 }
 
+func decodeImportRequest(pb *internal.ImportRequest, m *pilosa.ImportRequest) {
+	m.Index = pb.Index
+	m.Field = pb.Field
+	m.Shard = pb.Shard
+	m.RowIDs = pb.RowIDs
+	m.ColumnIDs = pb.ColumnIDs
+	m.RowKeys = pb.RowKeys
+	m.ColumnKeys = pb.ColumnKeys
+	m.Timestamps = pb.Timestamps
+}
+
+func decodeImportValueRequest(pb *internal.ImportValueRequest, m *pilosa.ImportValueRequest) {
+	m.Index = pb.Index
+	m.Field = pb.Field
+	m.Shard = pb.Shard
+	m.ColumnIDs = pb.ColumnIDs
+	m.ColumnKeys = pb.ColumnKeys
+	m.Values = pb.Values
+}
+
+func decodeImportResponse(pb *internal.ImportResponse, m *pilosa.ImportResponse) {
+	m.Err = pb.Err
+}
+
+func decodeBlockDataRequest(pb *internal.BlockDataRequest, m *pilosa.BlockDataRequest) {
+	m.Index = pb.Index
+	m.Field = pb.Field
+	m.View = pb.View
+	m.Shard = pb.Shard
+	m.Block = pb.Block
+}
+
+func decodeBlockDataResponse(pb *internal.BlockDataResponse, m *pilosa.BlockDataResponse) {
+	m.RowIDs = pb.RowIDs
+	m.ColumnIDs = pb.ColumnIDs
+}
+
 func decodeQueryResponse(pb *internal.QueryResponse, m *pilosa.QueryResponse) {
 	m.ColumnAttrSets = make([]*pilosa.ColumnAttrSet, len(pb.ColumnAttrSets))
 	decodeColumnAttrSets(pb.ColumnAttrSets, m.ColumnAttrSets)
diff --git a/handler.go b/handler.go
index 1f3d04300..9fc3af368 100644
--- a/handler.go
+++ b/handler.go
@@ -75,3 +75,40 @@ func (n nopHandler) Close() error {
 }
 
 var NopHandler Handler = nopHandler{}
+
+type ImportValueRequest struct {
+	Index      string
+	Field      string
+	Shard      uint64
+	ColumnIDs  []uint64
+	ColumnKeys []string
+	Values     []int64
+}
+
+type ImportRequest struct {
+	Index      string
+	Field      string
+	Shard      uint64
+	RowIDs     []uint64
+	ColumnIDs  []uint64
+	RowKeys    []string
+	ColumnKeys []string
+	Timestamps []int64
+}
+
+type ImportResponse struct {
+	Err string
+}
+
+type BlockDataRequest struct {
+	Index string
+	Field string
+	View  string
+	Shard uint64
+	Block uint64
+}
+
+type BlockDataResponse struct {
+	RowIDs    []uint64
+	ColumnIDs []uint64
+}
diff --git a/http/client.go b/http/client.go
index d8fb7f7ef..19bf82815 100644
--- a/http/client.go
+++ b/http/client.go
@@ -29,10 +29,8 @@ import (
 
 	"crypto/tls"
 
-	"github.com/gogo/protobuf/proto"
 	"github.com/pilosa/pilosa"
-	pilosaproto "github.com/pilosa/pilosa/encoding/proto"
-	"github.com/pilosa/pilosa/internal"
+	"github.com/pilosa/pilosa/encoding/proto"
 	"github.com/pkg/errors"
 )
 
@@ -68,7 +66,7 @@ func NewInternalClient(host string, remoteClient *http.Client) (*InternalClient,
 func NewInternalClientFromURI(defaultURI *pilosa.URI, remoteClient *http.Client) *InternalClient {
 	return &InternalClient{
 		defaultURI: defaultURI,
-		serializer: pilosaproto.Serializer{},
+		serializer: proto.Serializer{},
 		HTTPClient: remoteClient,
 	}
 }
@@ -282,7 +280,7 @@ func (c *InternalClient) Import(ctx context.Context, index, field string, shard
 		return pilosa.ErrFieldRequired
 	}
 
-	buf, err := marshalImportPayload(index, field, shard, bits)
+	buf, err := c.marshalImportPayload(index, field, shard, bits)
 	if err != nil {
 		return fmt.Errorf("Error Creating Payload: %s", err)
 	}
@@ -311,7 +309,7 @@ func (c *InternalClient) ImportK(ctx context.Context, index, field string, colum
 		return pilosa.ErrFieldRequired
 	}
 
-	buf, err := marshalImportPayloadK(index, field, columns)
+	buf, err := c.marshalImportPayloadK(index, field, columns)
 	if err != nil {
 		return fmt.Errorf("Error Creating Payload: %s", err)
 	}
@@ -345,14 +343,14 @@ func (c *InternalClient) EnsureField(ctx context.Context, indexName string, fiel
 }
 
 // marshalImportPayload marshalls the import parameters into a protobuf byte slice.
-func marshalImportPayload(index, field string, shard uint64, bits []pilosa.Bit) ([]byte, error) {
+func (c *InternalClient) marshalImportPayload(index, field string, shard uint64, bits []pilosa.Bit) ([]byte, error) {
 	// Separate row and column IDs to reduce allocations.
 	rowIDs := Bits(bits).RowIDs()
 	columnIDs := Bits(bits).ColumnIDs()
 	timestamps := Bits(bits).Timestamps()
 
 	// Marshal data to protobuf.
-	buf, err := proto.Marshal(&internal.ImportRequest{
+	buf, err := c.serializer.Marshal(&pilosa.ImportRequest{
 		Index:      index,
 		Field:      field,
 		Shard:      shard,
@@ -367,14 +365,14 @@ func marshalImportPayload(index, field string, shard uint64, bits []pilosa.Bit)
 }
 
 // marshalImportPayloadK marshalls the import parameters into a protobuf byte slice.
-func marshalImportPayloadK(index, field string, bits []pilosa.Bit) ([]byte, error) {
+func (c *InternalClient) marshalImportPayloadK(index, field string, bits []pilosa.Bit) ([]byte, error) {
 	// Separate row and column IDs to reduce allocations.
 	rowKeys := Bits(bits).RowKeys()
 	columnKeys := Bits(bits).ColumnKeys()
 	timestamps := Bits(bits).Timestamps()
 
 	// Marshal data to protobuf.
-	buf, err := proto.Marshal(&internal.ImportRequest{
+	buf, err := c.serializer.Marshal(&pilosa.ImportRequest{
 		Index:      index,
 		Field:      field,
 		RowKeys:    rowKeys,
@@ -416,8 +414,8 @@ func (c *InternalClient) importNode(ctx context.Context, node *pilosa.Node, inde
 		return errors.New(string(body))
 	}
 
-	var isresp internal.ImportResponse
-	if err := proto.Unmarshal(body, &isresp); err != nil {
+	var isresp pilosa.ImportResponse
+	if err := c.serializer.Unmarshal(body, &isresp); err != nil {
 		return fmt.Errorf("unmarshal import response: %s", err)
 	} else if s := isresp.Err; s != "" {
 		return errors.New(s)
@@ -434,7 +432,7 @@ func (c *InternalClient) ImportValue(ctx context.Context, index, field string, s
 		return pilosa.ErrFieldRequired
 	}
 
-	buf, err := marshalImportValuePayload(index, field, shard, vals)
+	buf, err := c.marshalImportValuePayload(index, field, shard, vals)
 	if err != nil {
 		return fmt.Errorf("Error Creating Payload: %s", err)
 	}
@@ -456,13 +454,13 @@ func (c *InternalClient) ImportValue(ctx context.Context, index, field string, s
 }
 
 // marshalImportValuePayload marshalls the import parameters into a protobuf byte slice.
-func marshalImportValuePayload(index, field string, shard uint64, vals []pilosa.FieldValue) ([]byte, error) {
+func (c *InternalClient) marshalImportValuePayload(index, field string, shard uint64, vals []pilosa.FieldValue) ([]byte, error) {
 	// Separate row and column IDs to reduce allocations.
 	columnIDs := FieldValues(vals).ColumnIDs()
 	values := FieldValues(vals).Values()
 
 	// Marshal data to protobuf.
-	buf, err := proto.Marshal(&internal.ImportValueRequest{
+	buf, err := c.serializer.Marshal(&pilosa.ImportValueRequest{
 		Index:     index,
 		Field:     field,
 		Shard:     shard,
@@ -685,7 +683,7 @@ func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index,
 	if uri == nil {
 		panic("need to pass a URI to BlockData")
 	}
-	buf, err := proto.Marshal(&internal.BlockDataRequest{
+	buf, err := c.serializer.Marshal(&pilosa.BlockDataRequest{
 		Index: index,
 		Field: field,
 		Shard: shard,
@@ -721,10 +719,10 @@ func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index,
 	}
 
 	// Decode response object.
-	var rsp internal.BlockDataResponse
+	var rsp pilosa.BlockDataResponse
 	if body, err := ioutil.ReadAll(resp.Body); err != nil {
 		return nil, nil, errors.Wrap(err, "reading")
-	} else if err := proto.Unmarshal(body, &rsp); err != nil {
+	} else if err := c.serializer.Unmarshal(body, &rsp); err != nil {
 		return nil, nil, errors.Wrap(err, "unmarshalling")
 	}
 	return rsp.RowIDs, rsp.ColumnIDs, nil
diff --git a/http/handler.go b/http/handler.go
index 33085879c..2b3d3ebd8 100644
--- a/http/handler.go
+++ b/http/handler.go
@@ -33,11 +33,9 @@ import (
 	"strings"
 	"time"
 
-	"github.com/gogo/protobuf/proto"
 	"github.com/gorilla/handlers"
 	"github.com/gorilla/mux"
 	"github.com/pilosa/pilosa"
-	"github.com/pilosa/pilosa/internal"
 
 	"github.com/pkg/errors"
 )
@@ -911,8 +909,8 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
 	if field.Type() == pilosa.FieldTypeInt {
 		// Field type: Int
 		// Marshal into request object.
-		var req internal.ImportValueRequest
-		if err := proto.Unmarshal(body, &req); err != nil {
+		req := &pilosa.ImportValueRequest{}
+		if err := h.API.Serializer.Unmarshal(body, req); err != nil {
 			http.Error(w, err.Error(), http.StatusBadRequest)
 			return
 		}
@@ -929,8 +927,8 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
 	} else {
 		// Field type: Set, Time
 		// Marshal into request object.
-		var req internal.ImportRequest
-		if err := proto.Unmarshal(body, &req); err != nil {
+		req := &pilosa.ImportRequest{}
+		if err := h.API.Serializer.Unmarshal(body, req); err != nil {
 			http.Error(w, err.Error(), http.StatusBadRequest)
 			return
 		}
@@ -947,7 +945,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Marshal response object.
-	buf, e := proto.Marshal(&internal.ImportResponse{Err: ""})
+	buf, e := h.API.Serializer.Marshal(&pilosa.ImportResponse{Err: ""})
 	if e != nil {
 		http.Error(w, fmt.Sprintf("marshal import response"), http.StatusInternalServerError)
 		return

From a164233c92f890c45bab13fa837073abe3d96feb Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Thu, 5 Jul 2018 16:21:27 -0500
Subject: [PATCH 21/32] fix handler tests not to use internal and fix bug

---
 encoding/proto/proto.go |  1 +
 server/handler_test.go  | 80 +++++++++++++++++++----------------------
 2 files changed, 37 insertions(+), 44 deletions(-)

diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go
index ed751247e..0020260ab 100644
--- a/encoding/proto/proto.go
+++ b/encoding/proto/proto.go
@@ -837,6 +837,7 @@ func decodeQueryResponse(pb *internal.QueryResponse, m *pilosa.QueryResponse) {
 
 func decodeColumnAttrSets(pb []*internal.ColumnAttrSet, m []*pilosa.ColumnAttrSet) {
 	for i := range pb {
+		m[i] = &pilosa.ColumnAttrSet{}
 		decodeColumnAttrSet(pb[i], m[i])
 	}
 }
diff --git a/server/handler_test.go b/server/handler_test.go
index e49b1eb0f..f0eeb18c8 100644
--- a/server/handler_test.go
+++ b/server/handler_test.go
@@ -27,10 +27,8 @@ import (
 
 	gohttp "net/http"
 
-	"github.com/gogo/protobuf/proto"
 	"github.com/pilosa/pilosa"
 	"github.com/pilosa/pilosa/http"
-	"github.com/pilosa/pilosa/internal"
 	"github.com/pilosa/pilosa/server"
 	"github.com/pilosa/pilosa/test"
 )
@@ -144,7 +142,7 @@ func TestHandler_Endpoints(t *testing.T) {
 
 	t.Run("Shards args protobuf", func(t *testing.T) {
 		// Generate request body.
-		reqBody, err := proto.Marshal(&internal.QueryRequest{
+		reqBody, err := cmd.API.Serializer.Marshal(&pilosa.QueryRequest{
 			Query:  "Count(Row(f0=30))",
 			Shards: []uint64{0, 1},
 		})
@@ -196,13 +194,11 @@ func TestHandler_Endpoints(t *testing.T) {
 			t.Fatalf("unexpected status code: %d", w.Code)
 		}
 
-		var resp internal.QueryResponse
-		if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+		var resp pilosa.QueryResponse
+		if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil {
 			t.Fatal(err)
-		} else if rt := resp.Results[0].Type; rt != http.QueryResultTypeUint64 {
-			t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
-		} else if n := resp.Results[0].N; n != 3 {
-			t.Fatalf("unexpected n: %d", n)
+		} else if rt, ok := resp.Results[0].(uint64); !ok || rt != 3 {
+			t.Fatalf("unexpected response type: %#v", resp.Results[0])
 		}
 	})
 
@@ -244,27 +240,25 @@ func TestHandler_Endpoints(t *testing.T) {
 			t.Fatalf("unexpected status code: %d", w.Code)
 		}
 
-		var resp internal.QueryResponse
-		if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+		var resp pilosa.QueryResponse
+		if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil {
 			t.Fatal(err)
-		} else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow {
-			t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
-		} else if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{pilosa.ShardWidth + 1, pilosa.ShardWidth + 2, (3 * pilosa.ShardWidth) + 4}) {
+		} else if columns := resp.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{pilosa.ShardWidth + 1, pilosa.ShardWidth + 2, (3 * pilosa.ShardWidth) + 4}) {
 			t.Fatalf("unexpected columns: %+v", columns)
-		} else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 {
+		} else if attrs := resp.Results[0].(*pilosa.Row).Attrs; len(attrs) != 3 {
 			t.Fatalf("unexpected attr length: %d", len(attrs))
-		} else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" {
-			t.Fatalf("unexpected attr[0]: %s=%v", k, v)
-		} else if k, v := attrs[1].Key, attrs[1].IntValue; k != "c" || v != int64(1) {
-			t.Fatalf("unexpected attr[1]: %s=%v", k, v)
-		} else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || !v {
-			t.Fatalf("unexpected attr[2]: %s=%v", k, v)
+		} else if attrs["a"] != "b" {
+			t.Fatalf("unexpected attr[a]: %v", attrs["a"])
+		} else if attrs["c"] != int64(1) {
+			t.Fatalf("unexpected attr[c]: %v", attrs["c"])
+		} else if !attrs["d"].(bool) {
+			t.Fatalf("unexpected attr[d]: %v", attrs["d"])
 		}
 	})
 
 	t.Run("Row columnattrs protobuf", func(t *testing.T) {
 		// Encode request body.
-		buf, err := proto.Marshal(&internal.QueryRequest{
+		buf, err := cmd.API.Serializer.Marshal(&pilosa.QueryRequest{
 			Query:       "Row(f0=30)",
 			ColumnAttrs: true,
 		})
@@ -281,22 +275,22 @@ func TestHandler_Endpoints(t *testing.T) {
 			t.Fatalf("unexpected status code: %d", w.Code)
 		}
 
-		var resp internal.QueryResponse
-		if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+		var resp pilosa.QueryResponse
+		if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil {
 			t.Fatal(err)
 		}
-		if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{pilosa.ShardWidth + 1, pilosa.ShardWidth + 2, (3 * pilosa.ShardWidth) + 4}) {
+		if columns := resp.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{pilosa.ShardWidth + 1, pilosa.ShardWidth + 2, (3 * pilosa.ShardWidth) + 4}) {
 			t.Fatalf("unexpected columns: %+v", columns)
-		} else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow {
-			t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
-		} else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 {
+		} else if _, ok := resp.Results[0].(*pilosa.Row); !ok {
+			t.Fatalf("unexpected response type: %#v", resp.Results[0])
+		} else if attrs := resp.Results[0].(*pilosa.Row).Attrs; len(attrs) != 3 {
 			t.Fatalf("unexpected attr length: %d", len(attrs))
-		} else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" {
-			t.Fatalf("unexpected attr[0]: %s=%v", k, v)
-		} else if k, v := attrs[1].Key, attrs[1].IntValue; k != "c" || v != int64(1) {
-			t.Fatalf("unexpected attr[1]: %s=%v", k, v)
-		} else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || !v {
-			t.Fatalf("unexpected attr[2]: %s=%v", k, v)
+		} else if attrs["a"] != "b" {
+			t.Fatalf("unexpected attr[a]: %v", attrs["a"])
+		} else if attrs["c"] != int64(1) {
+			t.Fatalf("unexpected attr[c]: %v", attrs["c"])
+		} else if !attrs["d"].(bool) {
+			t.Fatalf("unexpected attr[d]: %v", attrs["d"])
 		}
 
 		if a := resp.ColumnAttrSets; len(a) != 2 {
@@ -305,8 +299,8 @@ func TestHandler_Endpoints(t *testing.T) {
 			t.Fatalf("unexpected id: %d", a[0].ID)
 		} else if len(a[0].Attrs) != 1 {
 			t.Fatalf("unexpected column attr length: %d", len(a))
-		} else if k, v := a[0].Attrs[0].Key, a[0].Attrs[0].StringValue; k != "x" || v != "y" {
-			t.Fatalf("unexpected attr[0]: %s=%v", k, v)
+		} else if a[0].Attrs["x"] != "y" {
+			t.Fatalf("unexpected attr[x]: %v", a[0].Attrs["x"])
 		}
 	})
 
@@ -329,12 +323,10 @@ func TestHandler_Endpoints(t *testing.T) {
 			t.Fatalf("unexpected status code: %d", w.Code)
 		}
 
-		var resp internal.QueryResponse
-		if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+		var resp pilosa.QueryResponse
+		if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil {
 			t.Fatal(err)
-		} else if rt := resp.Results[0].Type; rt != http.QueryResultTypePairs {
-			t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
-		} else if a := resp.Results[0].GetPairs(); len(a) != 2 {
+		} else if a := resp.Results[0].([]pilosa.Pair); len(a) != 2 {
 			t.Fatalf("unexpected pair length: %d", len(a))
 		}
 	})
@@ -358,10 +350,10 @@ func TestHandler_Endpoints(t *testing.T) {
 			t.Fatalf("unexpected status code: %d", w.Code)
 		}
 
-		var resp internal.QueryResponse
-		if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+		var resp pilosa.QueryResponse
+		if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil {
 			t.Fatal(err)
-		} else if s := resp.Err; s != `executing: field not found` {
+		} else if s := resp.Err.Error(); s != `executing: field not found` {
 			t.Fatalf("unexpected error: %s", s)
 		}
 	})

From b7a583d6a8fc8ca687611d40885b1c59f7a5f0d6 Mon Sep 17 00:00:00 2001
From: Travis Turner <travis@pilosa.com>
Date: Thu, 5 Jul 2018 16:24:40 -0500
Subject: [PATCH 22/32] use Set() instead of SetValue() for integer fields

---
 executor.go      | 123 ++++++++++++++++++++---------------------------
 executor_test.go |  58 +++++++++++-----------
 pql/ast.go       |  20 ++++++++
 3 files changed, 102 insertions(+), 99 deletions(-)

diff --git a/executor.go b/executor.go
index 1a81e92bd..96e327f43 100644
--- a/executor.go
+++ b/executor.go
@@ -185,8 +185,6 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s
 		return e.executeCount(ctx, index, c, shards, opt)
 	case "Set":
 		return e.executeSetBit(ctx, index, c, opt)
-	case "SetValue":
-		return nil, e.executeSetValue(ctx, index, c, opt)
 	case "SetRowAttrs":
 		return nil, e.executeSetRowAttrs(ctx, index, c, opt)
 	case "SetColumnAttrs":
@@ -1077,14 +1075,7 @@ func (e *executor) executeSetBit(ctx context.Context, index string, c *pql.Call,
 		return false, ErrFieldNotFound
 	}
 
-	// Read fields using labels.
-	rowID, ok, err := c.UintArg(fieldName)
-	if err != nil {
-		return false, fmt.Errorf("reading Set() row: %v", err)
-	} else if !ok {
-		return false, fmt.Errorf("Set() row argument '%v' required", rowLabel)
-	}
-
+	// Read colID using labels.
 	colID, ok, err := c.UintArg("_" + columnLabel)
 	if err != nil {
 		return false, fmt.Errorf("reading Set() column: %v", err)
@@ -1092,20 +1083,40 @@ func (e *executor) executeSetBit(ctx context.Context, index string, c *pql.Call,
 		return false, fmt.Errorf("Set() column argument '%v' required", columnLabel)
 	}
 
-	var timestamp *time.Time
-	sTimestamp, ok := c.Args["_timestamp"].(string)
-	if ok {
-		t, err := time.Parse(TimeFormat, sTimestamp)
+	if f.Type() == FieldTypeInt {
+		// Read remaining fields using labels.
+		rowVal, ok, err := c.IntArg(fieldName)
 		if err != nil {
-			return false, fmt.Errorf("invalid date: %s", sTimestamp)
+			return false, fmt.Errorf("reading Set() row: %v", err)
+		} else if !ok {
+			return false, fmt.Errorf("Set() row argument '%v' required", rowLabel)
 		}
-		timestamp = &t
-	}
 
-	return e.executeSetBitField(ctx, index, c, f, colID, rowID, timestamp, opt)
+		return e.executeSetValueField(ctx, index, c, f, colID, rowVal, opt)
+	} else {
+		// Read remaining fields using labels.
+		rowID, ok, err := c.UintArg(fieldName)
+		if err != nil {
+			return false, fmt.Errorf("reading Set() row: %v", err)
+		} else if !ok {
+			return false, fmt.Errorf("Set() row argument '%v' required", rowLabel)
+		}
+
+		var timestamp *time.Time
+		sTimestamp, ok := c.Args["_timestamp"].(string)
+		if ok {
+			t, err := time.Parse(TimeFormat, sTimestamp)
+			if err != nil {
+				return false, fmt.Errorf("invalid date: %s", sTimestamp)
+			}
+			timestamp = &t
+		}
+
+		return e.executeSetBitField(ctx, index, c, f, colID, rowID, timestamp, opt)
+	}
 }
 
-// executeSetBitField executes a Set() call for a specific view.
+// executeSetBitField executes a Set() call for a specific field.
 func (e *executor) executeSetBitField(ctx context.Context, index string, c *pql.Call, f *Field, colID, rowID uint64, timestamp *time.Time, opt *execOptions) (bool, error) {
 	shard := colID / ShardWidth
 	ret := false
@@ -1137,64 +1148,36 @@ func (e *executor) executeSetBitField(ctx context.Context, index string, c *pql.
 	return ret, nil
 }
 
-// executeSetValue executes a SetValue() call.
-func (e *executor) executeSetValue(ctx context.Context, index string, c *pql.Call, opt *execOptions) error {
-	// Parse labels.
-	columnID, ok, err := c.UintArg(columnLabel)
-	if err != nil {
-		return fmt.Errorf("reading SetValue() column: %v", err)
-	} else if !ok {
-		return fmt.Errorf("SetValue() column field '%v' required", columnLabel)
-	}
+// executeSetValueField executes a Set() call for a specific int field.
+func (e *executor) executeSetValueField(ctx context.Context, index string, c *pql.Call, f *Field, colID uint64, value int64, opt *execOptions) (bool, error) {
+	shard := colID / ShardWidth
+	ret := false
 
-	// Copy args and remove reserved fields.
-	args := pql.CopyArgs(c.Args)
-	// While field could technically work as a ColumnAttr argument, we are treating it as a reserved word primarily to avoid confusion.
-	// Also, if we ever need to make ColumnAttrs field-specific, then having this reserved word prevents backward incompatibility.
-	delete(args, columnLabel)
-
-	// Set values.
-	for name, value := range args {
-		// Retrieve field.
-		field := e.Holder.Field(index, name)
-		if field == nil {
-			return ErrFieldNotFound
-		}
-
-		switch value := value.(type) {
-		case int64:
-			if _, err := field.SetValue(columnID, value); err != nil {
-				return err
+	for _, node := range e.Cluster.shardNodes(index, shard) {
+		// Update locally if host matches.
+		if node.ID == e.Node.ID {
+			val, err := f.SetValue(colID, value)
+			if err != nil {
+				return false, err
+			} else if val {
+				ret = true
 			}
-		default:
-			return ErrInvalidBSIGroupValueType
+			continue
 		}
-		field.Stats.Count("SetValue", 1, 1.0)
-	}
 
-	// Do not forward call if this is already being forwarded.
-	if opt.Remote {
-		return nil
-	}
+		// Do not forward call if this is already being forwarded.
+		if opt.Remote {
+			continue
+		}
 
-	// Execute on remote nodes in parallel.
-	nodes := Nodes(e.Cluster.Nodes).FilterID(e.Node.ID)
-	resp := make(chan error, len(nodes))
-	for _, node := range nodes {
-		go func(node *Node) {
-			_, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt)
-			resp <- err
-		}(node)
-	}
-
-	// Return first error.
-	for range nodes {
-		if err := <-resp; err != nil {
-			return err
+		// Forward call to remote node otherwise.
+		if res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt); err != nil {
+			return false, err
+		} else {
+			ret = res[0].(bool)
 		}
 	}
-
-	return nil
+	return ret, nil
 }
 
 // executeSetRowAttrs executes a SetRowAttrs() call.
diff --git a/executor_test.go b/executor_test.go
index c5e2f24a8..1bbeafddd 100644
--- a/executor_test.go
+++ b/executor_test.go
@@ -405,9 +405,9 @@ func TestExecutor_Execute_SetValue(t *testing.T) {
 		}
 
 		// Set bsiGroup values.
-		if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(col=10, f=25)`}); err != nil {
+		if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(10, f=25)`}); err != nil {
 			t.Fatal(err)
-		} else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(col=100, f=10)`}); err != nil {
+		} else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=10)`}); err != nil {
 			t.Fatal(err)
 		}
 
@@ -440,19 +440,19 @@ func TestExecutor_Execute_SetValue(t *testing.T) {
 		}
 
 		t.Run("ErrColumnBSIGroupRequired", func(t *testing.T) {
-			if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(invalid_column_name=10, f=100)`}); err == nil || errors.Cause(err).Error() != `SetValue() column field 'col' required` {
+			if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(invalid_column_name=10, f=100)`}); err == nil || errors.Cause(err).Error() != `field not found` {
 				t.Fatalf("unexpected error: %s", err)
 			}
 		})
 
 		t.Run("ErrColumnBSIGroupValue", func(t *testing.T) {
-			if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(invalid_column_name="bad_column", f=100)`}); err == nil || errors.Cause(err).Error() != `SetValue() column field 'col' required` {
+			if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set("bad_column", f=100)`}); err == nil || errors.Cause(err).Error() != `string 'col' value not allowed unless index 'keys' option enabled` {
 				t.Fatalf("unexpected error: %s", err)
 			}
 		})
 
 		t.Run("ErrInvalidBSIGroupValueType", func(t *testing.T) {
-			if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(col=10, f="hello")`}); err == nil || errors.Cause(err) != pilosa.ErrInvalidBSIGroupValueType {
+			if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(10, f="hello")`}); err == nil || errors.Cause(err).Error() != `string 'row' value not allowed unless field 'keys' option enabled` {
 				t.Fatalf("unexpected error: %s", err)
 			}
 		})
@@ -748,14 +748,14 @@ func TestExecutor_Execute_MinMax(t *testing.T) {
 		Set(1, x=1)
 		Set(` + strconv.Itoa(ShardWidth+2) + `, x=2)
 
-		SetValue(col=0, f=20)
-		SetValue(col=1, f=-5)
-		SetValue(col=2, f=-5)
-		SetValue(col=3, f=10)
-		SetValue(col=` + strconv.Itoa(ShardWidth) + `, f=30)
-		SetValue(col=` + strconv.Itoa(ShardWidth+2) + `, f=40)
-		SetValue(col=` + strconv.Itoa((5*ShardWidth)+100) + `, f=50)
-		SetValue(col=` + strconv.Itoa(ShardWidth+1) + `, f=60)
+		Set(0, f=20)
+		Set(1, f=-5)
+		Set(2, f=-5)
+		Set(3, f=10)
+		Set(` + strconv.Itoa(ShardWidth) + `, f=30)
+		Set(` + strconv.Itoa(ShardWidth+2) + `, f=40)
+		Set(` + strconv.Itoa((5*ShardWidth)+100) + `, f=50)
+		Set(` + strconv.Itoa(ShardWidth+1) + `, f=60)
 	`}); err != nil {
 		t.Fatal(err)
 	}
@@ -844,13 +844,13 @@ func TestExecutor_Execute_Sum(t *testing.T) {
 		Set(0, x=0)
 		Set(` + strconv.Itoa(ShardWidth+1) + `, x=0)
 
-		SetValue(col=0, foo=20)
-		SetValue(col=0, bar=2000)
-		SetValue(col=` + strconv.Itoa(ShardWidth) + `, foo=30)
-		SetValue(col=` + strconv.Itoa(ShardWidth+2) + `, foo=40)
-		SetValue(col=` + strconv.Itoa((5*ShardWidth)+100) + `, foo=50)
-		SetValue(col=` + strconv.Itoa(ShardWidth+1) + `, foo=60)
-		SetValue(col=0, other=1000)
+		Set(0, foo=20)
+		Set(0, bar=2000)
+		Set(` + strconv.Itoa(ShardWidth) + `, foo=30)
+		Set(` + strconv.Itoa(ShardWidth+2) + `, foo=40)
+		Set(` + strconv.Itoa((5*ShardWidth)+100) + `, foo=50)
+		Set(` + strconv.Itoa(ShardWidth+1) + `, foo=60)
+		Set(0, other=1000)
 	`}); err != nil {
 		t.Fatal(err)
 	}
@@ -959,15 +959,15 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) {
 		Set(0, f=0)
 		Set(` + strconv.Itoa(ShardWidth+1) + `, f=0)
 
-		SetValue(col=50, foo=20)
-		SetValue(col=50, bar=2000)
-		SetValue(col=` + strconv.Itoa(ShardWidth) + `, foo=30)
-		SetValue(col=` + strconv.Itoa(ShardWidth+2) + `, foo=10)
-		SetValue(col=` + strconv.Itoa((5*ShardWidth)+100) + `, foo=20)
-		SetValue(col=` + strconv.Itoa(ShardWidth+1) + `, foo=60)
-		SetValue(col=0, other=1000)
-		SetValue(col=0, edge=100)
-		SetValue(col=1, edge=-100)
+		Set(50, foo=20)
+		Set(50, bar=2000)
+		Set(` + strconv.Itoa(ShardWidth) + `, foo=30)
+		Set(` + strconv.Itoa(ShardWidth+2) + `, foo=10)
+		Set(` + strconv.Itoa((5*ShardWidth)+100) + `, foo=20)
+		Set(` + strconv.Itoa(ShardWidth+1) + `, foo=60)
+		Set(0, other=1000)
+		Set(0, edge=100)
+		Set(1, edge=-100)
 	`}); err != nil {
 		t.Fatal(err)
 	}
diff --git a/pql/ast.go b/pql/ast.go
index 0bcc582d4..4539bc95f 100644
--- a/pql/ast.go
+++ b/pql/ast.go
@@ -285,6 +285,26 @@ func (c *Call) UintArg(key string) (uint64, bool, error) {
 	}
 }
 
+// IntArg is for reading the value at key from call.Args as a uint64. If the
+// key is not in Call.Args, the value of the returned bool will be false, and
+// the error will be nil. The value is assumed to be a uint64 or an int64 and
+// then cast to a uint64. An error is returned if the value is not an int64 or
+// uint64.
+func (c *Call) IntArg(key string) (int64, bool, error) {
+	val, ok := c.Args[key]
+	if !ok {
+		return 0, false, nil
+	}
+	switch tval := val.(type) {
+	case int64:
+		return tval, true, nil
+	case uint64:
+		return int64(tval), true, nil
+	default:
+		return 0, true, fmt.Errorf("could not convert %v of type %T to uint64 in Call.IntArg", tval, tval)
+	}
+}
+
 // UintSliceArg reads the value at key from call.Args as a slice of uint64. If
 // the key is not in Call.Args, the value of the returned bool will be false,
 // and the error will be nil. If the value is a slice of int64 it will convert

From def3be0f17df00ccdfde76d829930f16aaf2e021 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Thu, 5 Jul 2018 16:25:04 -0500
Subject: [PATCH 23/32] remove more dead code

---
 cache.go  | 25 -------------------------
 pilosa.go | 19 -------------------
 2 files changed, 44 deletions(-)

diff --git a/cache.go b/cache.go
index 4c826f5d7..044d85f3c 100644
--- a/cache.go
+++ b/cache.go
@@ -22,7 +22,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/pilosa/pilosa/internal"
 	"github.com/pilosa/pilosa/lru"
 )
 
@@ -318,22 +317,6 @@ type Pair struct {
 	Count uint64 `json:"count"`
 }
 
-func encodePair(p Pair) *internal.Pair {
-	return &internal.Pair{
-		ID:    p.ID,
-		Key:   p.Key,
-		Count: p.Count,
-	}
-}
-
-func decodePair(pb *internal.Pair) Pair {
-	return Pair{
-		ID:    pb.ID,
-		Key:   pb.Key,
-		Count: pb.Count,
-	}
-}
-
 // Pairs is a sortable slice of Pair objects.
 type Pairs []Pair
 
@@ -409,14 +392,6 @@ func (p Pairs) String() string {
 	return buf.String()
 }
 
-func decodePairs(a []*internal.Pair) []Pair {
-	other := make([]Pair, len(a))
-	for i := range a {
-		other[i] = decodePair(a[i])
-	}
-	return other
-}
-
 // uint64Slice represents a sortable slice of uint64 numbers.
 type uint64Slice []uint64
 
diff --git a/pilosa.go b/pilosa.go
index ebc2be438..9615e88b8 100644
--- a/pilosa.go
+++ b/pilosa.go
@@ -17,8 +17,6 @@ package pilosa
 import (
 	"errors"
 	"regexp"
-
-	"github.com/pilosa/pilosa/internal"
 )
 
 // System errors.
@@ -122,23 +120,6 @@ type ColumnAttrSet struct {
 	Attrs map[string]interface{} `json:"attrs,omitempty"`
 }
 
-// EncodeColumnAttrSets converts a into its internal representation.
-func EncodeColumnAttrSets(a []*ColumnAttrSet) []*internal.ColumnAttrSet {
-	other := make([]*internal.ColumnAttrSet, len(a))
-	for i := range a {
-		other[i] = EncodeColumnAttrSet(a[i])
-	}
-	return other
-}
-
-// EncodeColumnAttrSet converts set into its internal representation.
-func EncodeColumnAttrSet(set *ColumnAttrSet) *internal.ColumnAttrSet {
-	return &internal.ColumnAttrSet{
-		ID:    set.ID,
-		Attrs: encodeAttrs(set.Attrs),
-	}
-}
-
 // TimeFormat is the go-style time format used to parse string dates.
 const TimeFormat = "2006-01-02T15:04"
 

From 462b27d9a9f8b1e8d50fab7ccd105e25a3bfbf22 Mon Sep 17 00:00:00 2001
From: Travis Turner <travis@pilosa.com>
Date: Thu, 5 Jul 2018 16:31:06 -0500
Subject: [PATCH 24/32] change executeSetBit to executeSet

---
 executor.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/executor.go b/executor.go
index 96e327f43..8ebc1df90 100644
--- a/executor.go
+++ b/executor.go
@@ -184,7 +184,7 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s
 		e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
 		return e.executeCount(ctx, index, c, shards, opt)
 	case "Set":
-		return e.executeSetBit(ctx, index, c, opt)
+		return e.executeSet(ctx, index, c, opt)
 	case "SetRowAttrs":
 		return nil, e.executeSetRowAttrs(ctx, index, c, opt)
 	case "SetColumnAttrs":
@@ -1058,8 +1058,8 @@ func (e *executor) executeClearBitField(ctx context.Context, index string, c *pq
 	return ret, nil
 }
 
-// executeSetBit executes a Set() call.
-func (e *executor) executeSetBit(ctx context.Context, index string, c *pql.Call, opt *execOptions) (bool, error) {
+// executeSet executes a Set() call.
+func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, opt *execOptions) (bool, error) {
 	fieldName, err := c.FieldArg()
 	if err != nil {
 		return false, errors.New("Set() argument required: field")

From 5717e32310912583c31f5e74ac7152b3f850ad02 Mon Sep 17 00:00:00 2001
From: Travis Turner <travis@pilosa.com>
Date: Thu, 5 Jul 2018 16:35:27 -0500
Subject: [PATCH 25/32] fix comment for IntArg

---
 pql/ast.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/pql/ast.go b/pql/ast.go
index 4539bc95f..dc16f4cdf 100644
--- a/pql/ast.go
+++ b/pql/ast.go
@@ -285,10 +285,10 @@ func (c *Call) UintArg(key string) (uint64, bool, error) {
 	}
 }
 
-// IntArg is for reading the value at key from call.Args as a uint64. If the
+// IntArg is for reading the value at key from call.Args as an int64. If the
 // key is not in Call.Args, the value of the returned bool will be false, and
-// the error will be nil. The value is assumed to be a uint64 or an int64 and
-// then cast to a uint64. An error is returned if the value is not an int64 or
+// the error will be nil. The value is assumed to be a unt64 or an int64 and
+// then cast to an int64. An error is returned if the value is not an int64 or
 // uint64.
 func (c *Call) IntArg(key string) (int64, bool, error) {
 	val, ok := c.Args[key]
@@ -301,7 +301,7 @@ func (c *Call) IntArg(key string) (int64, bool, error) {
 	case uint64:
 		return int64(tval), true, nil
 	default:
-		return 0, true, fmt.Errorf("could not convert %v of type %T to uint64 in Call.IntArg", tval, tval)
+		return 0, true, fmt.Errorf("could not convert %v of type %T to int64 in Call.IntArg", tval, tval)
 	}
 }
 

From d0485a3a1985a7eebd119c3f5b13240077207f44 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Thu, 5 Jul 2018 16:51:23 -0500
Subject: [PATCH 26/32] remove unused code in row.go

---
 row.go | 36 ------------------------------------
 1 file changed, 36 deletions(-)

diff --git a/row.go b/row.go
index 4d9ba3662..d6a0037cc 100644
--- a/row.go
+++ b/row.go
@@ -18,7 +18,6 @@ import (
 	"encoding/json"
 	"sort"
 
-	"github.com/pilosa/pilosa/internal"
 	"github.com/pilosa/pilosa/roaring"
 )
 
@@ -252,41 +251,6 @@ func (r *Row) Columns() []uint64 {
 	return a
 }
 
-// Union performs a union on a slice of rows.
-func Union(rows []*Row) *Row {
-	other := rows[0]
-	for _, r := range rows[1:] {
-		other = other.Union(r)
-	}
-	return other
-}
-
-// EncodeRow converts r into its internal representation.
-func EncodeRow(r *Row) *internal.Row {
-	if r == nil {
-		return nil
-	}
-
-	return &internal.Row{
-		Columns: r.Columns(),
-		Attrs:   encodeAttrs(r.Attrs),
-	}
-}
-
-// DecodeRow converts r from its internal representation.
-func DecodeRow(pr *internal.Row) *Row {
-	if pr == nil {
-		return nil
-	}
-
-	r := NewRow()
-	r.Attrs = decodeAttrs(pr.Attrs)
-	for _, v := range pr.Columns {
-		r.SetBit(v)
-	}
-	return r
-}
-
 // RowSegment holds a subset of a row.
 // This could point to a mmapped roaring bitmap or an in-memory bitmap. The
 // width of the segment will always match the shard width.

From e33682cfd27a3d1efdaf6fb8888e3cc9d66f5bea Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Thu, 5 Jul 2018 17:38:49 -0500
Subject: [PATCH 27/32] address feedback

---
 cluster.go             | 4 ++--
 utils_internal_test.go | 3 +--
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/cluster.go b/cluster.go
index 6d8cf38b4..f61173c3b 100644
--- a/cluster.go
+++ b/cluster.go
@@ -459,7 +459,7 @@ func (c *cluster) receiveNodeState(nodeID string, state string) error {
 	return nil
 }
 
-// Status returns the the cluster's status including what nodes it contains, it's ID, and current state.
+// Status returns the the cluster's status including what nodes it contains, its ID, and current state.
 func (c *cluster) Status() *ClusterStatus {
 	return &ClusterStatus{
 		ClusterID: c.id,
@@ -1764,7 +1764,7 @@ type ResizeSource struct {
 	Shard uint64 `protobuf:"varint,5,opt,name=Shard,proto3" json:"Shard,omitempty"`
 }
 
-// Schema is a schema
+// Schema contains information about indexes and their configuration.
 type Schema struct {
 	Indexes []*IndexInfo
 }
diff --git a/utils_internal_test.go b/utils_internal_test.go
index e0a42bdc1..df1b7d2e0 100644
--- a/utils_internal_test.go
+++ b/utils_internal_test.go
@@ -364,8 +364,7 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *ResizeInstruction) error
 		}
 
 		for _, src := range instr.Sources {
-			srcNode := src.Node
-			srcCluster := t.clusterByID(srcNode.ID)
+			srcCluster := t.clusterByID(src.Node.ID)
 
 			srcFragment := srcCluster.holder.fragment(src.Index, src.Field, src.View, src.Shard)
 			destFragment := destCluster.holder.fragment(src.Index, src.Field, src.View, src.Shard)

From 21cf6b6e57b1cd18c26a8a51609f452cab49f482 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Thu, 5 Jul 2018 17:59:18 -0500
Subject: [PATCH 28/32] remove URI getters since the fields were exported for
 serialization

---
 gossip/gossip.go     |  6 +++---
 http/client.go       |  4 ++--
 server/server.go     | 12 ++++++------
 uri.go               | 15 ---------------
 uri_internal_test.go | 14 +++++++-------
 5 files changed, 18 insertions(+), 33 deletions(-)

diff --git a/gossip/gossip.go b/gossip/gossip.go
index 28b3ac690..7a11d01d4 100644
--- a/gossip/gossip.go
+++ b/gossip/gossip.go
@@ -146,7 +146,7 @@ func WithLogger(logger *log.Logger) GossipMemberSetOption {
 
 // NewGossipMemberSet returns a new instance of GossipMemberSet based on options.
 func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...GossipMemberSetOption) (*GossipMemberSet, error) {
-	host := api.Node().URI.GetHost()
+	host := api.Node().URI.Host
 	g := &GossipMemberSet{
 		papi:   api,
 		Logger: pilosa.NopLogger,
@@ -191,10 +191,10 @@ func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...GossipMemberSetO
 	conf := memberlist.DefaultWANConfig()
 	conf.Transport = g.transport.Net
 	conf.Name = api.Node().ID
-	conf.BindAddr = api.Node().URI.GetHost()
+	conf.BindAddr = api.Node().URI.Host
 	conf.BindPort = port
 	conf.AdvertisePort = port
-	conf.AdvertiseAddr = hostToIP(api.Node().URI.GetHost())
+	conf.AdvertiseAddr = hostToIP(api.Node().URI.Host)
 	//
 	conf.TCPTimeout = time.Duration(cfg.StreamTimeout)
 	conf.SuspicionMult = cfg.SuspicionMult
diff --git a/http/client.go b/http/client.go
index 19bf82815..de6eb89f6 100644
--- a/http/client.go
+++ b/http/client.go
@@ -993,7 +993,7 @@ func pos(rowID, columnID uint64) uint64 {
 
 func uriPathToURL(uri *pilosa.URI, path string) url.URL {
 	return url.URL{
-		Scheme: uri.GetScheme(),
+		Scheme: uri.Scheme,
 		Host:   uri.HostPort(),
 		Path:   path,
 	}
@@ -1001,7 +1001,7 @@ func uriPathToURL(uri *pilosa.URI, path string) url.URL {
 
 func nodePathToURL(node *pilosa.Node, path string) url.URL {
 	return url.URL{
-		Scheme: node.URI.GetScheme(),
+		Scheme: node.URI.Scheme,
 		Host:   node.URI.HostPort(),
 		Path:   path,
 	}
diff --git a/server/server.go b/server/server.go
index f46f6a64a..401da09e8 100644
--- a/server/server.go
+++ b/server/server.go
@@ -203,7 +203,7 @@ func (m *Command) SetupServer() error {
 
 	// Setup TLS
 	var TLSConfig *tls.Config
-	if uri.GetScheme() == "https" {
+	if uri.Scheme == "https" {
 		if m.Config.TLS.CertificatePath == "" {
 			return errors.New("certificate path is required for TLS sockets")
 		}
@@ -236,7 +236,7 @@ func (m *Command) SetupServer() error {
 	}
 
 	// If port is 0, get auto-allocated port from listener
-	if uri.GetPort() == 0 {
+	if uri.Port == 0 {
 		uri.SetPort(uint16(m.ln.Addr().(*net.TCPAddr).Port))
 	}
 
@@ -311,7 +311,7 @@ func (m *Command) SetupNetworking() error {
 	}
 
 	// get the host portion of addr to use for binding
-	gossipHost := m.API.Node().URI.GetHost()
+	gossipHost := m.API.Node().URI.Host
 	m.gossipTransport, err = gossip.NewTransport(gossipHost, gossipPort, m.logger.Logger())
 	if err != nil {
 		return errors.Wrap(err, "getting transport")
@@ -368,19 +368,19 @@ func NewStatsClient(name string, host string) (pilosa.StatsClient, error) {
 // getListener gets a net.Listener based on the config.
 func getListener(uri pilosa.URI, tlsconf *tls.Config) (ln net.Listener, err error) {
 	// If bind URI has the https scheme, enable TLS
-	if uri.GetScheme() == "https" && tlsconf != nil {
+	if uri.Scheme == "https" && tlsconf != nil {
 		ln, err = tls.Listen("tcp", uri.HostPort(), tlsconf)
 		if err != nil {
 			return nil, errors.Wrap(err, "tls.Listener")
 		}
-	} else if uri.GetScheme() == "http" {
+	} else if uri.Scheme == "http" {
 		// Open HTTP listener to determine port (if specified as :0).
 		ln, err = net.Listen("tcp", uri.HostPort())
 		if err != nil {
 			return nil, errors.Wrap(err, "net.Listen")
 		}
 	} else {
-		return nil, errors.Errorf("unsupported scheme: %s", uri.GetScheme())
+		return nil, errors.Errorf("unsupported scheme: %s", uri.Scheme)
 	}
 
 	return ln, nil
diff --git a/uri.go b/uri.go
index e01bf8bcc..8577c8238 100644
--- a/uri.go
+++ b/uri.go
@@ -82,11 +82,6 @@ func NewURIFromAddress(address string) (*URI, error) {
 	return parseAddress(address)
 }
 
-// GetScheme returns the scheme of this URI.
-func (u *URI) GetScheme() string {
-	return u.Scheme
-}
-
 // SetScheme sets the scheme of this URI.
 func (u *URI) SetScheme(scheme string) error {
 	m := schemeRegexp.FindStringSubmatch(scheme)
@@ -97,11 +92,6 @@ func (u *URI) SetScheme(scheme string) error {
 	return nil
 }
 
-// GetHost returns the host of this URI.
-func (u *URI) GetHost() string {
-	return u.Host
-}
-
 // SetHost sets the host of this URI.
 func (u *URI) SetHost(host string) error {
 	m := hostRegexp.FindStringSubmatch(host)
@@ -112,11 +102,6 @@ func (u *URI) SetHost(host string) error {
 	return nil
 }
 
-// GetPort returns the port of this URI.
-func (u *URI) GetPort() uint16 {
-	return u.Port
-}
-
 // SetPort sets the port of this URI.
 func (u *URI) SetPort(port uint16) {
 	u.Port = port
diff --git a/uri_internal_test.go b/uri_internal_test.go
index 2587223f8..3c9631661 100644
--- a/uri_internal_test.go
+++ b/uri_internal_test.go
@@ -83,8 +83,8 @@ func TestSetScheme(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if uri.GetScheme() != target {
-		t.Fatalf("%s != %s", uri.GetScheme(), target)
+	if uri.Scheme != target {
+		t.Fatalf("%s != %s", uri.Scheme, target)
 	}
 }
 
@@ -95,7 +95,7 @@ func TestSetHost(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if uri.GetHost() != target {
+	if uri.Host != target {
 		t.Fatalf("%s != %s", uri.Host, target)
 	}
 }
@@ -104,7 +104,7 @@ func TestSetPort(t *testing.T) {
 	uri := DefaultURI()
 	target := uint16(9999)
 	uri.SetPort(target)
-	if uri.GetPort() != target {
+	if uri.Port != target {
 		t.Fatalf("%d != %d", uri.Port, target)
 	}
 }
@@ -137,13 +137,13 @@ func TestHostPort(t *testing.T) {
 }
 
 func compare(t *testing.T, uri *URI, scheme string, host string, port uint16) {
-	if uri.GetScheme() != scheme {
+	if uri.Scheme != scheme {
 		t.Fatalf("Scheme does not match: %s != %s", uri.Scheme, scheme)
 	}
-	if uri.GetHost() != host {
+	if uri.Host != host {
 		t.Fatalf("Host does not match: %s != %s", uri.Host, host)
 	}
-	if uri.GetPort() != port {
+	if uri.Port != port {
 		t.Fatalf("Port does not match: %d != %d", uri.Port, port)
 	}
 }

From 7873126720ff6b125796b56e44cc1c40cb74a831 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Thu, 5 Jul 2018 18:39:25 -0500
Subject: [PATCH 29/32] handle errors a bit better in handlePostQuery

---
 http/handler.go | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/http/handler.go b/http/handler.go
index 7786eb2f3..8cc52a9b9 100644
--- a/http/handler.go
+++ b/http/handler.go
@@ -412,18 +412,25 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) {
 
 	resp, err := h.API.Query(r.Context(), req)
 	if err != nil {
-		w.WriteHeader(http.StatusBadRequest)
+		switch errors.Cause(resp.Err) {
+		case pilosa.ErrTooManyWrites:
+			w.WriteHeader(http.StatusRequestEntityTooLarge)
+		default:
+			w.WriteHeader(http.StatusBadRequest)
+		}
 		h.writeQueryResponse(w, r, &pilosa.QueryResponse{Err: err})
 		return
 	}
 
-	// Set appropriate status code, if there is an error.
+	// Set appropriate status code, if there is an error. It doesn't appear that
+	// resp.Err could ever be set in API.Query, so this code block is probably
+	// doing nothing right now.
 	if resp.Err != nil {
-		switch resp.Err {
+		switch errors.Cause(resp.Err) {
 		case pilosa.ErrTooManyWrites:
 			w.WriteHeader(http.StatusRequestEntityTooLarge)
 		default:
-			w.WriteHeader(http.StatusInternalServerError)
+			w.WriteHeader(http.StatusBadRequest)
 		}
 	}
 

From 9305712237935229f562af2c034a9c192b5486df Mon Sep 17 00:00:00 2001
From: Travis Turner <travis@pilosa.com>
Date: Thu, 5 Jul 2018 18:02:31 -0500
Subject: [PATCH 30/32] add options.keys and lowercase names to json output

---
 field.go | 14 ++++++++++----
 index.go | 15 +++++++++------
 2 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/field.go b/field.go
index e1eed501d..4cf1861df 100644
--- a/field.go
+++ b/field.go
@@ -1094,9 +1094,9 @@ func (f *Field) ImportValue(columnIDs []uint64, values []int64) error {
 
 func (f *Field) MarshalJSON() ([]byte, error) {
 	thing := struct {
-		Name    string
-		Options FieldOptions
-		Views   []*ViewInfo
+		Name    string       `json:"name"`
+		Options FieldOptions `json:"options"`
+		Views   []*ViewInfo  `json:"views"`
 	}{
 		Name:    f.Name(),
 		Options: f.Options(),
@@ -1134,7 +1134,7 @@ type FieldOptions struct {
 	Min         int64       `json:"min,omitempty"`
 	Max         int64       `json:"max,omitempty"`
 	TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"`
-	Keys        bool        `json:"keys,omitempty"`
+	Keys        bool        `json:"keys"`
 }
 
 // applyDefaultOptions returns a new FieldOptions object
@@ -1177,28 +1177,34 @@ func (o *FieldOptions) MarshalJSON() ([]byte, error) {
 			Type      string `json:"type"`
 			CacheType string `json:"cacheType"`
 			CacheSize uint32 `json:"cacheSize"`
+			Keys      bool   `json:"keys"`
 		}{
 			o.Type,
 			o.CacheType,
 			o.CacheSize,
+			o.Keys,
 		})
 	case FieldTypeInt:
 		return json.Marshal(struct {
 			Type string `json:"type"`
 			Min  int64  `json:"min"`
 			Max  int64  `json:"max"`
+			Keys bool   `json:"keys"`
 		}{
 			o.Type,
 			o.Min,
 			o.Max,
+			o.Keys,
 		})
 	case FieldTypeTime:
 		return json.Marshal(struct {
 			Type        string      `json:"type"`
 			TimeQuantum TimeQuantum `json:"timeQuantum"`
+			Keys        bool        `json:"keys"`
 		}{
 			o.Type,
 			o.TimeQuantum,
+			o.Keys,
 		})
 	}
 	return nil, errors.New("invalid field type")
diff --git a/index.go b/index.go
index 9d27f4174..0f6d222be 100644
--- a/index.go
+++ b/index.go
@@ -83,11 +83,13 @@ func (i *Index) MarshalJSON() ([]byte, error) {
 		fields = append(fields, f)
 	}
 	thing := struct {
-		Name   string
-		Fields []*Field
+		Name    string       `json:"name"`
+		Options IndexOptions `json:"options"`
+		Fields  []*Field     `json:"fields"`
 	}{
-		Name:   i.name,
-		Fields: fields,
+		Name:    i.name,
+		Options: i.Options(),
+		Fields:  fields,
 	}
 	return json.Marshal(thing)
 }
@@ -422,8 +424,9 @@ func (p indexSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() }
 
 // IndexInfo represents schema information for an index.
 type IndexInfo struct {
-	Name   string       `json:"name"`
-	Fields []*FieldInfo `json:"fields"`
+	Name    string       `json:"name"`
+	Options IndexOptions `json:"options"`
+	Fields  []*FieldInfo `json:"fields"`
 }
 
 type indexInfoSlice []*IndexInfo

From c10cdc9d222f1f045f822108386bb86d6cbfb147 Mon Sep 17 00:00:00 2001
From: Travis Turner <travis@pilosa.com>
Date: Thu, 5 Jul 2018 19:15:34 -0500
Subject: [PATCH 31/32] exclude views from http schema output

---
 api.go          |  6 +++---
 holder.go       | 16 ++++++++++++++++
 http/handler.go |  2 +-
 3 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/api.go b/api.go
index 7a1a6576a..8fe028675 100644
--- a/api.go
+++ b/api.go
@@ -482,9 +482,9 @@ func (api *API) ClusterMessage(ctx context.Context, reqBody io.Reader) error {
 }
 
 // Schema returns information about each index in Pilosa including which fields
-// and views they contain.
-func (api *API) Schema(ctx context.Context) []*Index {
-	return api.holder.Indexes()
+// they contain.
+func (api *API) Schema(ctx context.Context) []*IndexInfo {
+	return api.holder.limitedSchema()
 }
 
 // Views returns the views in the given field.
diff --git a/holder.go b/holder.go
index 2481f4768..192c83a46 100644
--- a/holder.go
+++ b/holder.go
@@ -228,6 +228,22 @@ func (h *Holder) Schema() []*IndexInfo {
 	return a
 }
 
+// limitedSchema returns schema information for all indexes and fields.
+func (h *Holder) limitedSchema() []*IndexInfo {
+	var a []*IndexInfo
+	for _, index := range h.Indexes() {
+		di := &IndexInfo{Name: index.Name()}
+		for _, field := range index.Fields() {
+			fi := &FieldInfo{Name: field.Name(), Options: field.Options()}
+			di.Fields = append(di.Fields, fi)
+		}
+		sort.Sort(fieldInfoSlice(di.Fields))
+		a = append(a, di)
+	}
+	sort.Sort(indexInfoSlice(a))
+	return a
+}
+
 // applySchema applies an internal Schema to Holder.
 func (h *Holder) applySchema(schema *Schema) error {
 	// Create indexes that don't exist.
diff --git a/http/handler.go b/http/handler.go
index 7786eb2f3..6dce53bc2 100644
--- a/http/handler.go
+++ b/http/handler.go
@@ -463,7 +463,7 @@ func (h *Handler) handleGetIndex(w http.ResponseWriter, r *http.Request) {
 	}
 	indexName := mux.Vars(r)["index"]
 	for _, idx := range h.API.Schema(r.Context()) {
-		if idx.Name() == indexName {
+		if idx.Name == indexName {
 			if err := json.NewEncoder(w).Encode(idx); err != nil {
 				h.Logger.Printf("write response error: %s", err)
 			}

From bf2b4e9284d3e9724d8c1f79fff05f4f1b40b9f4 Mon Sep 17 00:00:00 2001
From: Travis Turner <travis@pilosa.com>
Date: Thu, 5 Jul 2018 20:50:00 -0500
Subject: [PATCH 32/32] remove index and field MarshalJSON

---
 field.go | 15 ---------------
 index.go | 19 -------------------
 2 files changed, 34 deletions(-)

diff --git a/field.go b/field.go
index 4cf1861df..bcdb800f8 100644
--- a/field.go
+++ b/field.go
@@ -1092,21 +1092,6 @@ func (f *Field) ImportValue(columnIDs []uint64, values []int64) error {
 	return nil
 }
 
-func (f *Field) MarshalJSON() ([]byte, error) {
-	thing := struct {
-		Name    string       `json:"name"`
-		Options FieldOptions `json:"options"`
-		Views   []*ViewInfo  `json:"views"`
-	}{
-		Name:    f.Name(),
-		Options: f.Options(),
-	}
-	for _, viewname := range f.viewNames() {
-		thing.Views = append(thing.Views, &ViewInfo{Name: viewname})
-	}
-	return json.Marshal(thing)
-}
-
 type fieldSlice []*Field
 
 func (p fieldSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
diff --git a/index.go b/index.go
index 0f6d222be..7217bb659 100644
--- a/index.go
+++ b/index.go
@@ -15,7 +15,6 @@
 package pilosa
 
 import (
-	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -76,24 +75,6 @@ func NewIndex(path, name string) (*Index, error) {
 	}, nil
 }
 
-func (i *Index) MarshalJSON() ([]byte, error) {
-	fields := make([]*Field, 0, len(i.fields))
-	for _, f := range i.fields {
-
-		fields = append(fields, f)
-	}
-	thing := struct {
-		Name    string       `json:"name"`
-		Options IndexOptions `json:"options"`
-		Fields  []*Field     `json:"fields"`
-	}{
-		Name:    i.name,
-		Options: i.Options(),
-		Fields:  fields,
-	}
-	return json.Marshal(thing)
-}
-
 // Name returns name of the index.
 func (i *Index) Name() string { return i.name }