Merge pull request #1573 from codysoyland/release-v1.0.2

Release v1.0.2

CHANGELOG.md

@@ -5,6 +5,33 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [v1.0.2] - 2018-08-01
+
+This version contains 11 contributions from 3 contributors. There are 30 files changed, 1,569 insertions, and 1,215 deletions.
+
+### Fixed
+
+- Fix documentation ([#1503](https://github.com/pilosa/pilosa/pull/1503), [#1495](https://github.com/pilosa/pilosa/pull/1495), [#1551](https://github.com/pilosa/pilosa/pull/1551))
+- Fix places where empty IndexOptions were being used ([#1547](https://github.com/pilosa/pilosa/pull/1547))
+- Fix translator syncing bug in cluster environments ([#1552](https://github.com/pilosa/pilosa/pull/1552))
+- Fix race condition in translate_test ([#1541](https://github.com/pilosa/pilosa/pull/1541))
+- Add IndexOptions to IndexInfo json response ([#1547](https://github.com/pilosa/pilosa/pull/1547))
+- Add proper locking to cluster code to prevent races ([#1533](https://github.com/pilosa/pilosa/pull/1533))
+- Re-export erroneously unexported func Row.Intersect ([#1502](https://github.com/pilosa/pilosa/pull/1502))
+- Update parser to handle row keys on SetRowAttrs() ([#1555](https://github.com/pilosa/pilosa/pull/1555))
+
+## [v1.0.1] - 2018-07-11
+
+This version contains 12 contributions from 4 contributors. There are 11 files changed, 133 insertions, and 39 deletions.
+
+### Fixed
+
+- Use `dep ensure -vendor-only` for build repeatability ([#1491](https://github.com/pilosa/pilosa/pull/1491))
+- Make sure time range views are calculated correctly across months ([#1485](https://github.com/pilosa/pilosa/pull/1485))
+- Fix up error handling, add a configurable timeout to http handler closing ([#1486](https://github.com/pilosa/pilosa/pull/1486))
+- Add gossip Closer ([#1483](https://github.com/pilosa/pilosa/pull/1483))
+- Update docs references to WebUI naming (console) and installation ([#1493](https://github.com/pilosa/pilosa/pull/1493))
+
 ## [v1.0.0] - 2018-07-09
 
 This version contains 218 contributions from 7 contributors. There are 184 files changed, 21,769 insertions, and 20,275 deletions.
@@ -55,7 +82,7 @@ This version contains 218 contributions from 7 contributors. There are 184 files
 
 ## [v0.10.0] - 2018-05-15
 
-This version contains 93 contribution from 8 contributors. There are 93 files changed, 4,495 insertions, and 5,392 deletions.
+This version contains 93 contributions from 8 contributors. There are 93 files changed, 4,495 insertions, and 5,392 deletions.
 
 ### Added
 
@@ -86,7 +113,7 @@ This version contains 93 contribution from 8 contributors. There are 93 files ch
 
 ## [v0.9.0] - 2018-05-04
 
-This version contains 188 contribution from 12 contributors. There are 141 files changed, 17,832 insertions, and 7,503 deletions.
+This version contains 188 contributions from 12 contributors. There are 141 files changed, 17,832 insertions, and 7,503 deletions.
 
 *Please see special [upgrading instructions](https://www.pilosa.com/docs/latest/administration/#version-0-9) for this release.*
 

api.go

@@ -135,9 +135,9 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er
 		}
 
 		// Translate column attributes, if necessary.
-		if api.server.primaryTranslateStore != nil {
+		if api.server.translateFile != nil {
 			for _, col := range resp.ColumnAttrSets {
-				v, err := api.server.primaryTranslateStore.TranslateColumnToString(req.Index, col.ID)
+				v, err := api.server.translateFile.TranslateColumnToString(req.Index, col.ID)
 				if err != nil {
 					return resp, err
 				}
@@ -756,46 +756,17 @@ func (api *API) ResizeAbort() error {
 	return errors.Wrap(err, "complete current job")
 }
 
-// translateStoreBufferSize is the buffer size used for streaming data.
-const translateStoreBufferSize = 65536
-
-func (api *API) GetTranslateData(ctx context.Context, w io.WriteCloser, offset int64) error {
-	rc, err := api.server.primaryTranslateStore.Reader(ctx, offset)
+// GetTranslateData provides a reader for key translation logs starting at offset.
+func (api *API) GetTranslateData(ctx context.Context, offset int64) (io.ReadCloser, error) {
+	rc, err := api.server.translateFile.Reader(ctx, offset)
 	if err != nil {
-		return errors.Wrap(err, "read from translate store")
+		return nil, errors.Wrap(err, "read from translate store")
 	}
 
 	// Ensure reader is closed when the client disconnects.
 	go func() { <-ctx.Done(); rc.Close() }()
 
-	go func() {
-		defer rc.Close()
-		defer w.Close()
-
-		buf := make([]byte, translateStoreBufferSize)
-
-		// Copy from reader to client until store or client disconnect.
-		for {
-			// Read from store.
-			n, err := rc.Read(buf)
-			if err == io.EOF {
-				return
-			} else if err != nil {
-				api.server.logger.Printf("api: translate store read error: %s", err)
-				return
-			} else if n == 0 {
-				continue
-			}
-
-			// Write to response & flush.
-			if _, err := w.Write(buf[:n]); err != nil {
-				api.server.logger.Printf("api: translate store response write error: %s", err)
-				return
-			}
-		}
-	}()
-
-	return nil
+	return rc, nil
 }
 
 // State returns the cluster state which is usually "NORMAL", but could be

cluster.go

@@ -233,8 +233,14 @@ func newCluster() *cluster {
 	}
 }
 
-// coordinatorNode returns the coordinator node.
 func (c *cluster) coordinatorNode() *Node {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	return c.unprotectedCoordinatorNode()
+}
+
+// unprotectedCoordinatorNode returns the coordinator node.
+func (c *cluster) unprotectedCoordinatorNode() *Node {
 	return c.unprotectedNodeByID(c.Coordinator)
 }
 
@@ -275,7 +281,7 @@ func (c *cluster) setCoordinator(n *Node) error {
 	}
 
 	// Broadcast cluster status.
-	return c.broadcaster.SendSync(c.Status())
+	return c.broadcaster.SendSync(c.status())
 }
 
 // updateCoordinator updates this nodes Coordinator value as well as
@@ -305,7 +311,7 @@ func (c *cluster) unprotectedUpdateCoordinator(n *Node) bool {
 }
 
 // addNode adds a node to the Cluster and updates and saves the
-// new topology.
+// new topology. unprotected.
 func (c *cluster) addNode(node *Node) error {
 	c.logger.Printf("add node %s to cluster on %s", node, c.Node)
 
@@ -332,7 +338,7 @@ func (c *cluster) addNode(node *Node) error {
 }
 
 // removeNode removes a node from the Cluster and updates and saves the
-// new topology.
+// new topology. unprotected.
 func (c *cluster) removeNode(node *Node) error {
 	// remove from cluster
 	if !c.removeNodeBasicSorted(node) {
@@ -356,7 +362,7 @@ func (c *cluster) nodeIDs() []string {
 	return Nodes(c.Nodes).IDs()
 }
 
-func (c *cluster) setID(id string) {
+func (c *cluster) unprotectedSetID(id string) {
 	// Don't overwrite ClusterID.
 	if c.id != "" {
 		return
@@ -375,11 +381,11 @@ func (c *cluster) State() string {
 
 func (c *cluster) SetState(state string) {
 	c.mu.Lock()
-	c.setState(state)
+	c.unprotectedSetState(state)
 	c.mu.Unlock()
 }
 
-func (c *cluster) setState(state string) {
+func (c *cluster) unprotectedSetState(state string) {
 	// Ignore cases where the state hasn't changed.
 	if state == c.state {
 		return
@@ -439,28 +445,38 @@ func (c *cluster) setNodeState(state string) error {
 // Coordinator to keep track of, during startup, which nodes have
 // finished opening their Holder.
 func (c *cluster) receiveNodeState(nodeID string, state string) error {
-	if !c.isCoordinator() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if !c.unprotectedIsCoordinator() {
 		return nil
 	}
 
 	// This method is really only useful during initial startup.
-	if c.State() != ClusterStateStarting {
+	if c.state != ClusterStateStarting {
 		return nil
 	}
 
+	c.Topology.mu.Lock()
 	c.Topology.nodeStates[nodeID] = state
+	c.Topology.mu.Unlock()
 	c.logger.Printf("received state %s (%s)", state, nodeID)
 
 	// Set cluster state to NORMAL.
 	if c.haveTopologyAgreement() && c.allNodesReady() {
-		return c.setStateAndBroadcast(ClusterStateNormal)
+		return c.unprotectedSetStateAndBroadcast(ClusterStateNormal)
 	}
 
 	return nil
 }
 
-// Status returns the the cluster's status including what nodes it contains, its ID, and current state.
-func (c *cluster) Status() *ClusterStatus {
+func (c *cluster) status() *ClusterStatus {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	return c.unprotectedStatus()
+}
+
+// unprotectedStatus returns the the cluster's status including what nodes it contains, its ID, and current state.
+func (c *cluster) unprotectedStatus() *ClusterStatus {
 	return &ClusterStatus{
 		ClusterID: c.id,
 		State:     c.state,
@@ -494,8 +510,8 @@ func (c *cluster) nodePositionByID(nodeID string) int {
 	return -1
 }
 
-// addNodeBasicSorted adds a node to the cluster, sorted by id.
-// Returns a pointer to the node and true if the node was added.
+// addNodeBasicSorted adds a node to the cluster, sorted by id. Returns a
+// pointer to the node and true if the node was added. unprotected.
 func (c *cluster) addNodeBasicSorted(node *Node) bool {
 	n := c.unprotectedNodeByID(node.ID)
 	if n != nil {
@@ -510,8 +526,8 @@ func (c *cluster) addNodeBasicSorted(node *Node) bool {
 	return true
 }
 
-// removeNodeBasicSorted removes a node from the cluster, maintaining
-// the sort order. Returns true if the node was removed.
+// removeNodeBasicSorted removes a node from the cluster, maintaining the sort
+// order. Returns true if the node was removed. unprotected.
 func (c *cluster) removeNodeBasicSorted(node *Node) bool {
 	i := c.nodePositionByID(node.ID)
 	if i < 0 {
@@ -599,7 +615,7 @@ func (c *cluster) fragCombos(idx string, maxShard uint64, fieldViews viewsByFiel
 
 // diff compares c with another cluster and determines if a node is being
 // added or removed. An error is returned for any case other than where
-// exactly one node is added or removed.
+// exactly one node is added or removed. unprotected.
 func (c *cluster) diff(other *cluster) (action string, nodeID string, err error) {
 	lenFrom := len(c.Nodes)
 	lenTo := len(other.Nodes)
@@ -638,7 +654,7 @@ func (c *cluster) diff(other *cluster) (action string, nodeID string, err error)
 }
 
 // fragSources returns a list of ResizeSources - for each node in the `to` cluster -
-// required to move from cluster `c` to cluster `to`.
+// required to move from cluster `c` to cluster `to`. unprotected.
 func (c *cluster) fragSources(to *cluster, idx *Index) (map[string][]*ResizeSource, error) {
 	m := make(map[string][]*ResizeSource)
 
@@ -737,7 +753,7 @@ func (c *cluster) partition(index string, shard uint64) int {
 	return int(h.Sum64() % uint64(c.partitionN))
 }
 
-// shardNodes returns a list of nodes that own a fragment.
+// shardNodes returns a list of nodes that own a fragment. unprotected
 func (c *cluster) shardNodes(index string, shard uint64) []*Node {
 	return c.partitionNodes(c.partition(index, shard))
 }
@@ -747,7 +763,7 @@ func (c *cluster) ownsShard(nodeID string, index string, shard uint64) bool {
 	return Nodes(c.shardNodes(index, shard)).ContainsID(nodeID)
 }
 
-// partitionNodes returns a list of nodes that own a partition.
+// partitionNodes returns a list of nodes that own a partition. unprotected.
 func (c *cluster) partitionNodes(partitionID int) []*Node {
 	// Default replica count to between one and the number of nodes.
 	// The replica count can be zero if there are no nodes.
@@ -887,10 +903,12 @@ func (c *cluster) markAsJoined() {
 	}
 }
 
+// needTopologyAgreement is unprotected.
 func (c *cluster) needTopologyAgreement() bool {
-	return c.State() == ClusterStateStarting && !stringSlicesAreEqual(c.Topology.nodeIDs, c.nodeIDs())
+	return c.state == ClusterStateStarting && !stringSlicesAreEqual(c.Topology.nodeIDs, c.nodeIDs())
 }
 
+// haveTopologyAgreement is unprotected.
 func (c *cluster) haveTopologyAgreement() bool {
 	if c.Static {
 		return true
@@ -898,6 +916,7 @@ func (c *cluster) haveTopologyAgreement() bool {
 	return stringSlicesAreEqual(c.Topology.nodeIDs, c.nodeIDs())
 }
 
+// allNodesReady is unprotected.
 func (c *cluster) allNodesReady() bool {
 	if c.Static {
 		return true
@@ -911,7 +930,10 @@ func (c *cluster) allNodesReady() bool {
 }
 
 func (c *cluster) handleNodeAction(nodeAction nodeAction) error {
-	j, err := c.generateResizeJob(nodeAction)
+
+	c.mu.Lock()
+	j, err := c.unprotectedGenerateResizeJob(nodeAction)
+	c.mu.Unlock()
 	if err != nil {
 		c.logger.Printf("generateResizeJob error: err=%s", err)
 		if err := c.setStateAndBroadcast(ClusterStateNormal); err != nil {
@@ -945,8 +967,12 @@ func (c *cluster) handleNodeAction(nodeAction nodeAction) error {
 		}
 		// Add/remove uri to/from the cluster.
 		if j.action == resizeJobActionRemove {
+			c.mu.Lock()
+			defer c.mu.Unlock()
 			return c.removeNode(nodeAction.node)
 		} else if j.action == resizeJobActionAdd {
+			c.mu.Lock()
+			defer c.mu.Unlock()
 			return c.addNode(nodeAction.node)
 		}
 	case resizeJobStateAborted:
@@ -957,14 +983,21 @@ func (c *cluster) handleNodeAction(nodeAction nodeAction) error {
 	return nil
 }
 
-func (c *cluster) setStateAndBroadcast(state string) error {
-	c.SetState(state)
+func (c *cluster) setStateAndBroadcast(state string) error { // nolint: unparam
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.unprotectedSetStateAndBroadcast(state)
+}
+
+func (c *cluster) unprotectedSetStateAndBroadcast(state string) error {
+	c.unprotectedSetState(state)
 	if c.Static {
 		return nil
 	}
 	// Broadcast cluster status changes to the cluster.
 	c.logger.Printf("broadcasting ClusterStatus: %s", state)
-	return c.broadcaster.SendSync(c.Status())
+	return c.broadcaster.SendSync(c.unprotectedStatus()) // TODO fix c.Status
+
 }
 
 func (c *cluster) sendTo(node *Node, m Message) error {
@@ -1027,15 +1060,13 @@ func (c *cluster) listenForJoins() {
 	}()
 }
 
-// generateResizeJob creates a new resizeJob based on the new node being
+// unprotectedGenerateResizeJob creates a new resizeJob based on the new node being
 // added/removed. It also saves a reference to the resizeJob in the `jobs` map
 // for future lookup by JobID.
-func (c *cluster) generateResizeJob(nodeAction nodeAction) (*resizeJob, error) {
+func (c *cluster) unprotectedGenerateResizeJob(nodeAction nodeAction) (*resizeJob, error) {
 	c.logger.Printf("generateResizeJob: %v", nodeAction)
-	c.mu.Lock()
-	defer c.mu.Unlock()
 
-	j, err := c.generateResizeJobByAction(nodeAction)
+	j, err := c.unprotectedGenerateResizeJobByAction(nodeAction)
 	if err != nil {
 		return nil, errors.Wrap(err, "generating job")
 	}
@@ -1053,11 +1084,11 @@ func (c *cluster) generateResizeJob(nodeAction nodeAction) (*resizeJob, error) {
 	return j, nil
 }
 
-// generateResizeJobByAction returns a resizeJob with instructions based on
+// unprotectedGenerateResizeJobByAction returns a resizeJob with instructions based on
 // the difference between Cluster and a new Cluster with/without uri.
 // Broadcaster is associated to the resizeJob here for use in broadcasting
 // the resize instructions to other nodes in the cluster.
-func (c *cluster) generateResizeJobByAction(nodeAction nodeAction) (*resizeJob, error) {
+func (c *cluster) unprotectedGenerateResizeJobByAction(nodeAction nodeAction) (*resizeJob, error) {
 	j := newResizeJob(c.Nodes, nodeAction.node, nodeAction.action)
 	j.Broadcaster = c.broadcaster
 
@@ -1101,10 +1132,10 @@ func (c *cluster) generateResizeJobByAction(nodeAction nodeAction) (*resizeJob,
 		instr := &ResizeInstruction{
 			JobID:         j.ID,
 			Node:          toCluster.unprotectedNodeByID(id),
-			Coordinator:   c.coordinatorNode(),
+			Coordinator:   c.unprotectedCoordinatorNode(),
 			Sources:       sources,
 			Schema:        &Schema{Indexes: c.holder.Schema()}, // Include the schema to ensure it's in sync on the receiving node.
-			ClusterStatus: c.Status(),
+			ClusterStatus: c.unprotectedStatus(),
 		}
 		j.Instructions = append(j.Instructions, instr)
 	}
@@ -1117,6 +1148,10 @@ func (c *cluster) generateResizeJobByAction(nodeAction nodeAction) (*resizeJob,
 func (c *cluster) completeCurrentJob(state string) error {
 	c.mu.Lock()
 	defer c.mu.Unlock()
+	return c.unprotectedCompleteCurrentJob(state)
+}
+
+func (c *cluster) unprotectedCompleteCurrentJob(state string) error {
 	if !c.unprotectedIsCoordinator() {
 		return ErrNodeNotCoordinator
 	}
@@ -1468,7 +1503,7 @@ func (t *Topology) encode() *internal.Topology {
 	return encodeTopology(t)
 }
 
-// loadTopology reads the topology for the node.
+// loadTopology reads the topology for the node. unprotected.
 func (c *cluster) loadTopology() error {
 	buf, err := ioutil.ReadFile(filepath.Join(c.Path, ".topology"))
 	if os.IsNotExist(err) {
@@ -1491,7 +1526,7 @@ func (c *cluster) loadTopology() error {
 	return nil
 }
 
-// saveTopology writes the current topology to disk.
+// saveTopology writes the current topology to disk. unprotected.
 func (c *cluster) saveTopology() error {
 
 	if err := os.MkdirAll(c.Path, 0777); err != nil {
@@ -1563,6 +1598,8 @@ func (c *cluster) ReceiveEvent(e *NodeEvent) error {
 }
 
 func (c *cluster) nodeJoin(node *Node) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
 	if c.needTopologyAgreement() {
 		// A host that is not part of the topology can't be added to the STARTING cluster.
 		if !c.Topology.ContainsID(node.ID) {
@@ -1582,7 +1619,7 @@ func (c *cluster) nodeJoin(node *Node) error {
 			// If the result of the previous AddNode completed the joining of nodes
 			// in the topology, then change the state to NORMAL.
 			if c.haveTopologyAgreement() {
-				return c.setStateAndBroadcast(ClusterStateNormal)
+				return c.unprotectedSetStateAndBroadcast(ClusterStateNormal)
 			}
 			return nil
 		} else if err != nil {
@@ -1590,11 +1627,11 @@ func (c *cluster) nodeJoin(node *Node) error {
 		}
 
 		if c.haveTopologyAgreement() && c.allNodesReady() {
-			return c.setStateAndBroadcast(ClusterStateNormal)
+			return c.unprotectedSetStateAndBroadcast(ClusterStateNormal)
 		} else {
 			// Send the status to the remote node. This lets the remote node
 			// know that it can proceed with opening its Holder.
-			return c.sendTo(node, c.Status())
+			return c.sendTo(node, c.unprotectedStatus())
 		}
 	}
 
@@ -1602,7 +1639,7 @@ func (c *cluster) nodeJoin(node *Node) error {
 	// This is useful in the case where a node is restarted or temporarily leaves
 	// the cluster.
 	if node := c.unprotectedNodeByID(node.ID); node != nil {
-		return c.sendTo(node, c.Status())
+		return c.sendTo(node, c.unprotectedStatus())
 	}
 
 	// If the holder does not yet contain data, go ahead and add the node.
@@ -1610,14 +1647,14 @@ func (c *cluster) nodeJoin(node *Node) error {
 		if err := c.addNode(node); err != nil {
 			return errors.Wrap(err, "adding node")
 		}
-		return c.setStateAndBroadcast(ClusterStateNormal)
+		return c.unprotectedSetStateAndBroadcast(ClusterStateNormal)
 	} else if err != nil {
 		return errors.Wrap(err, "checking if holder has data2")
 	}
 
 	// If the cluster has data, we need to change to RESIZING and
 	// kick off the resizing process.
-	if err := c.setStateAndBroadcast(ClusterStateResizing); err != nil {
+	if err := c.unprotectedSetStateAndBroadcast(ClusterStateResizing); err != nil {
 		return errors.Wrap(err, "broadcasting state")
 	}
 	c.joiningLeavingNodes <- nodeAction{node, resizeJobActionAdd}
@@ -1627,13 +1664,15 @@ func (c *cluster) nodeJoin(node *Node) error {
 
 // nodeLeave initiates the removal of a node from the cluster.
 func (c *cluster) nodeLeave(node *Node) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
 	// Refuse the request if this is not the coordinator.
-	if !c.isCoordinator() {
-		return fmt.Errorf("node removal requests are only valid on the coordinator node: %s", c.coordinatorNode().ID)
+	if !c.unprotectedIsCoordinator() {
+		return fmt.Errorf("node removal requests are only valid on the coordinator node: %s", c.unprotectedCoordinatorNode().ID)
 	}
 
-	if c.State() != ClusterStateNormal {
-		return fmt.Errorf("Cluster must be in state %s to remove a node. Current state: %s", ClusterStateNormal, c.State())
+	if c.state != ClusterStateNormal {
+		return fmt.Errorf("Cluster must be in state %s to remove a node. Current state: %s", ClusterStateNormal, c.state)
 	}
 
 	// Ensure that node is in the cluster.
@@ -1647,7 +1686,7 @@ func (c *cluster) nodeLeave(node *Node) error {
 	}
 
 	// See if resize job can be generated
-	if _, err := c.generateResizeJobByAction(nodeAction{c.unprotectedNodeByID(node.ID), resizeJobActionRemove}); err != nil {
+	if _, err := c.unprotectedGenerateResizeJobByAction(nodeAction{c.unprotectedNodeByID(node.ID), resizeJobActionRemove}); err != nil {
 		return errors.Wrap(err, "generating job")
 	}
 
@@ -1664,14 +1703,14 @@ func (c *cluster) nodeLeave(node *Node) error {
 		if err := c.removeNode(n); err != nil {
 			return errors.Wrap(err, "removing node")
 		}
-		return c.setStateAndBroadcast(ClusterStateNormal)
+		return c.unprotectedSetStateAndBroadcast(ClusterStateNormal)
 	} else if err != nil {
 		return errors.Wrap(err, "checking if holder has data")
 	}
 
 	// If the cluster has data then change state to RESIZING and
 	// kick off the resizing process.
-	if err := c.setStateAndBroadcast(ClusterStateResizing); err != nil {
+	if err := c.unprotectedSetStateAndBroadcast(ClusterStateResizing); err != nil {
 		return errors.Wrap(err, "broadcasting state")
 	}
 	c.joiningLeavingNodes <- nodeAction{n, resizeJobActionRemove}
@@ -1689,7 +1728,7 @@ func (c *cluster) mergeClusterStatus(cs *ClusterStatus) error {
 	}
 
 	// Set ClusterID.
-	c.setID(cs.ClusterID)
+	c.unprotectedSetID(cs.ClusterID)
 
 	officialNodes := cs.Nodes
 
@@ -1721,13 +1760,15 @@ func (c *cluster) mergeClusterStatus(cs *ClusterStatus) error {
 		}
 	}
 
-	c.setState(cs.State)
+	c.unprotectedSetState(cs.State)
 
 	c.markAsJoined()
 
 	return nil
 }
 
+// setStatic is unprotected, but only called before the cluster has been started
+// (and therefore not concurrently).
 func (c *cluster) setStatic(hosts []string) error {
 	c.Static = true
 	c.Coordinator = c.Node.ID

docs/api-reference.md

@@ -17,7 +17,7 @@ Returns the schema of all indexes in JSON.
 curl -XGET localhost:10101/index
 ```
 ``` response
-{"indexes":[{"name":"user","fields":[{"name":"collab"}]}]}
+{"indexes":[{"name":"user","fields":[{"name":"event","options":{"type":"time","timeQuantum":"YMD","keys":false}}]}]}
 ```
 
 ### List index schema
@@ -30,7 +30,7 @@ Returns the schema of the specified index in JSON.
 curl -XGET localhost:10101/index/user
 ```
 ``` response
-{"name":"user", "fields":[{"name":"collab"}]}
+{"name":"user","fields":[{"name":"event","options":{"type":"time","timeQuantum":"YMD","keys":false}}]}
 ```
 
 ### Create index
@@ -100,19 +100,27 @@ By default, all bits and attributes (*for `Row` queries only*) are returned. In
 
 Creates a field in the given index with the given name.
 
-The request payload is in JSON, and may contain the `options` field. The `options` field is a JSON object which may contain the following fields:
+The request payload is in JSON, and may contain the `options` field. The `options` field is a JSON object which must contain a `type` along with the corresponding configuration options. 
 
-* `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this field.
-* `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this field. Default is `lru`.
-* `cacheSize` (int): Number of rows to keep in the cache. Default 50,000.
-* `fields` (array): List of range-encoded [fields](../data-model/#bsi-range-encoding).
+* `set`
+    * `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this field. Default is `lru`.
+    * `cacheSize` (int): Number of rows to keep in the cache. Default 50,000.
+* `int`
+    * `min` (int): Minimum integer value allowed for the field.
+    * `max` (int): Maximum integer value allowed for the field.
+* `time`
+    * `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this field.
 
-Each individual `field` contains the following:
+The following example creates an `int` field called "quantity" capable of storing values from -1000 to 2000:
 
-* `name` (string): Field name.
-* `type` (string): Field type, "set", "int" or "time".
-* `min` (int): Minimum value allowed for this field.
-* `max` (int): Maximum value allowed for this field.
+``` request
+curl localhost:10101/index/user/field/quantity \
+     -X POST \
+     -d '{"options": {"type": "int", "min": -1000, "max":2000}}'
+```
+``` response
+{"success":true}
+```
 
 Integer fields are stored as n-bit range-encoded values. Pilosa supports 63-bit, signed integers with values between `min` and `max`.
 

docs/data-model.md

@@ -18,11 +18,11 @@ nav = [
 
 ### Overview
 
-The central component of Pilosa's data model is a boolean matrix. Each cell in the matrix is a single bit - if the bit is set, it indicates that a relationship exists between that particular row and column.
+The central component of Pilosa's data model is a boolean matrix. Each cell in the matrix is a single bit; if the bit is set, it indicates that a relationship exists between that particular row and column.
 
-Rows and columns can represent anything (they could even represent the same set of things - a [bigraph](https://en.wikipedia.org/wiki/Bigraph)). Pilosa can associate arbitrary key/value pairs (referred to as attributes) to rows and columns, but queries and storage are optimized around the core matrix.
+Rows and columns can represent anything (they could even represent the same set of things as in a [bigraph](https://en.wikipedia.org/wiki/Bigraph)). Pilosa can associate arbitrary key/value pairs (referred to as attributes) to rows and columns, but queries and storage are optimized around the core matrix.
 
-Pilosa lays out data first in rows, so queries which get all the set bits in one or many rows, or compute a combining operation on multiple rows such as Intersect or Union are the fastest. Pilosa categorizes rows into different *fields* and quickly retrieves the top rows in a field sorted by the number of columns set in each row.
+Pilosa lays out data first in rows, so queries which get all the set bits in one or many rows, or compute a combining operation—such as Intersect or Union—on multiple rows, are the fastest. Pilosa categorizes rows into different *fields* and quickly retrieves the top rows in a field sorted by the number of columns set in each row.
 
 Please note that Pilosa is most performant when row and column IDs are sequential starting from 0. You can deviate from this to some degree, but setting a bit with column ID 2<sup>63</sup> on a single-node cluster, for example, will not work well due to memory limitations.
 
@@ -35,11 +35,11 @@ The purpose of the Index is to represent a data namespace. You cannot perform cr
 
 ### Column
 
-Column ids are sequential increasing integers and are common to all Fields within an Index. A single column often corresponds to a record in a relational table, although other configurations are possible, and sometimes preferable.
+Column ids are sequential, increasing integers and they are common to all Fields within an Index. A single column often corresponds to a record in a relational table, although other configurations are possible, and sometimes preferable.
 
 ### Row
 
-Row ids are sequential increasing integers namespaced to each Field within an Index.
+Row ids are sequential, increasing integers namespaced to each Field within an Index.
 
 ### Field
 
@@ -49,8 +49,6 @@ Fields are used to segment rows within an index, for example to define different
 
 The Pilosa index is a flexible structure; it can represent any sort of high-cardinality binary matrix. We have explored a number of modeling patterns in Pilosa use cases; one accessible example is a direct analogy to the relational model, summarized here.
 
-TODO diagram showing a few rows of a relational table and corresponding pilosa index
-
 Entities:
 
  Relational  | Pilosa
@@ -103,7 +101,7 @@ The LRU cache maintains the most recently accessed Rows.
 
 ### Time Quantum
 
-Setting a time quantum on a field creates extra views which allow Range queries down to the time interval specified. For example - if the time quantum is set to `YMD`, Range queries down to the granularity of a day are supported.
+Setting a time quantum on a field creates extra views which allow Range queries down to the time interval specified. For example, if the time quantum is set to `YMD`, Range queries down to the granularity of a day are supported.
 
 ### Attribute
 
@@ -117,27 +115,36 @@ Indexes are segmented into groups of columns called shards (previously known as
 
 Query operations run in parallel, and they are evenly distributed across a cluster via a consistent hash algorithm.
 
-### View
+### Field Type
 
-Views represent the various data layouts within a Field. The primary View is called Standard, and it contains the typical Row and Column data. Time-based Views are automatically generated for each time quantum. Views are internally managed by Pilosa, and never exposed directly via the API.
+Upon creation, fields are configured to be of a certain type. Pilosa supports three field types: `set`, `int`, and `time`.
 
-#### Standard
+#### Set
 
-The standard View contains the same Row/Column format as the input data. 
-
-#### Time Quantums
-
-If a Field has a time quantum, then Views are generated for each of the defined time segments. For example, for a field with a time quantum of `YMD`, the following `Set()` queries will result in the data described in the diagram below:
+Set is the default field type in Pilosa. Set fields represent a standard, binary matrix of rows and columns where each row key represents a possible field value. The following example creates a `set` field called "info" with a ranked cache containing up to 100,000 records.
 
+``` request
+curl localhost:10101/index/repository/field/info \
+     -X POST \
+     -d '{"options": {"type": "set", "cacheType": "ranked", "cacheSize":100000}}'
 ```
-Set(3, A=8, 2017-05-18T00:00)
-Set(3, A=8, 2017-05-19T00:00)
+``` response
+{"success":true}
 ```
 
-![time quantum field diagram](/img/docs/field-time-quantum.svg)
-*Time quantum fueld diagram*
+#### Int
+Fields of type `int` are used to store integer values. Integer fields share the same columns as the other fields in the index, but values for the field must be integers that fall between the `min` and `max` values specified when creating the field. The following example creates an `int` field called "quantity" capable of storing values from -1000 to 2000:
 
-#### BSI Range-Encoding
+``` request
+curl localhost:10101/index/repository/field/quantity \
+     -X POST \
+     -d '{"options": {"type": "int", "min": -1000, "max":2000}}'
+```
+``` response
+{"success":true}
+```
+
+##### BSI Range-Encoding
 
 Bit-Sliced Indexing (BSI) is the storage method Pilosa uses to represent multi-bit integers in a bitmap index. Integers are stored as n-bit, range-encoded bit-sliced indexes of base-2, along with an additional row indicating "not null". This means that a 16-bit integer will require 17 rows: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null row. Pilosa can evaluate `Range`, `Min`, `Max`, and `Sum` queries on these BSI integers. The result of a `Sum` query includes a count, which can be used to compute an average with no other overhead.
 
@@ -158,3 +165,26 @@ Set(3, B=6)
 *BSI field diagram*
 
 Check out this [blog post](/blog/range-encoded-bitmaps/) for some more details about BSI in Pilosa.
+
+#### Time
+
+Time fields are similar to `set` fields, but in addition to row and column information, they also store a per-bit time value down to a defined granularity. The following example creates a `time` field called "event" which stores timestamp information down to a day granularity.
+
+``` request
+curl localhost:10101/index/repository/field/event \
+     -X POST \
+     -d '{"options": {"type": "time", "timeQuantum": "YMD"}}'
+```
+``` response
+{"success":true}
+```
+
+With `time` fields, data views are generated for each of the defined time segments. For example, for a field with a time quantum of `YMD`, the following `Set()` queries will result in the data described in the diagram below:
+
+```
+Set(3, A=8, 2017-05-18T00:00)
+Set(3, A=8, 2017-05-19T00:00)
+```
+
+![time quantum field diagram](/img/docs/field-time-quantum.svg)
+*Time quantum fueld diagram*

docs/getting-started.md

@@ -19,7 +19,7 @@ Any HTTP tool can be used to interact with the Pilosa server. The examples in th
 
 ### Starting Pilosa
 
-Follow the steps in the [Install](../installation/) document to install Pilosa.
+Follow the steps in the [Installation](../installation/) document to install Pilosa.
 Execute the following in a terminal to run Pilosa with the default configuration (Pilosa will be available at [localhost:10101](http://localhost:10101)):
 ```
 pilosa server

docs/glossary.md

@@ -14,7 +14,7 @@ nav = []
 
 <strong id="bitmap">[Bitmap](../data-model/#overview):</strong> The on-disk and in-memory representation of a [row](#row). Implemented with [Roaring](#roaring-bitmap).
 
-<strong id="bsi">[BSI](../data-model/#bsi-range-encoding)</strong> Bit-sliced indexing is the method Pilosa uses to represent multi-bit integers. Integer values are stored in [fields](#field), and can be used for [Range](#range-bsi), [Min](#min), [Max](#max), and [Sum](#sum) queries.
+<strong id="bsi">[BSI](../data-model/#bsi-range-encoding)</strong> Bit-sliced indexing is the method Pilosa uses to represent multi-bit integers. Integer values are stored in `int` [fields](#field), and can be used for [Range](#range-bsi), [Min](#min), [Max](#max), and [Sum](#sum) queries.
 
 <strong id="cluster">Cluster:</strong> A cluster consists of one or more [nodes](#node) which share a cluster configuration. The cluster also defines how data is [replicated](#replica) and how internode communication is coordinated. Pilosa does not have a leader node, all data is evenly distributed, and any node can respond to queries.
 

docs/installation.md

@@ -42,7 +42,7 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
     backing up, and more. Complete documentation is available
     at https://www.pilosa.com/docs/.
 
-    Version: v1.0.0
+    Version: v1.0.2
     Build Time: 2018-05-14T22:14:01+0000
 
     Usage:
@@ -71,19 +71,19 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
 
 1. Download the latest release:
     ```
-    curl -L -O https://github.com/pilosa/pilosa/releases/download/v1.0.0/pilosa-v1.0.0-darwin-amd64.tar.gz
+    curl -L -O https://github.com/pilosa/pilosa/releases/download/v1.0.2/pilosa-v1.0.2-darwin-amd64.tar.gz
     ```
 
     Other releases can be downloaded from our Releases page on Github.
 
 2. Extract the binary:
     ```
-    tar xfz pilosa-v1.0.0-darwin-amd64.tar.gz
+    tar xfz pilosa-v1.0.2-darwin-amd64.tar.gz
     ```
 
 3. Move the binary into your PATH so you can run `pilosa` from any shell:
     ```
-    cp -i pilosa-v1.0.0-darwin-amd64/pilosa /usr/local/bin
+    cp -i pilosa-v1.0.2-darwin-amd64/pilosa /usr/local/bin
     ```
 
 4. Make sure Pilosa is installed successfully:
@@ -100,7 +100,7 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
     backing up, and more. Complete documentation is available
     at https://www.pilosa.com/docs/.
 
-    Version: v1.0.0
+    Version: v1.0.2
     Build Time: 2018-05-14T22:14:01+0000
 
     Usage:
@@ -163,7 +163,7 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
     backing up, and more. Complete documentation is available
     at https://www.pilosa.com/docs/.
 
-    Version: v1.0.0
+    Version: v1.0.2
     Build Time: 2018-05-14T22:14:01+0000
 
     Usage:
@@ -222,19 +222,19 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
 
 1. To install the latest version of Pilosa, download the latest release:
     ```
-    curl -L -O https://github.com/pilosa/pilosa/releases/download/v1.0.0/pilosa-v1.0.0-linux-amd64.tar.gz
+    curl -L -O https://github.com/pilosa/pilosa/releases/download/v1.0.2/pilosa-v1.0.2-linux-amd64.tar.gz
     ```
 
     Note: This assumes you are using an `amd64` compatible architecture. Other releases can be downloaded from our Releases page on Github.
 
 2. Extract the binary:
     ```
-    tar xfz pilosa-v1.0.0-linux-amd64.tar.gz
+    tar xfz pilosa-v1.0.2-linux-amd64.tar.gz
     ```
 
 3. Move the binary into your PATH so you can run `pilosa` from any shell:
     ```
-    cp -i pilosa-v1.0.0-linux-amd64/pilosa /usr/local/bin
+    cp -i pilosa-v1.0.2-linux-amd64/pilosa /usr/local/bin
     ```
 
 4. Make sure Pilosa is installed successfully:
@@ -251,7 +251,7 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
     backing up, and more. Complete documentation is available
     at https://www.pilosa.com/docs/.
 
-    Version: v1.0.0
+    Version: v1.0.2
     Build Time: 2018-05-14T22:14:01+0000
 
     Usage:
@@ -314,7 +314,7 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
     backing up, and more. Complete documentation is available
     at https://www.pilosa.com/docs/.
 
-    Version: v1.0.0
+    Version: v1.0.2
     Build Time: 2018-05-14T22:14:01+0000
 
     Usage:

docs/introduction.md

@@ -8,7 +8,7 @@ nav = []
 ## Introduction
 
 
-Pilosa is an open source, distributed bitmap index.
+Pilosa is an open source, distributed index.
 
 [//]: # (TODO insert a graphic here?)
 

docs/query-language.md

@@ -1,4 +1,4 @@
-v+++
++++
 title = "Query Language"
 weight = 6
 nav = [
@@ -419,7 +419,7 @@ Count(Row(stargazer=1))
 **Spec:**
 
 ```
-TopN([ROW_CALL], <FIELD>, [n=UINT],
+TopN(<FIELD>, [ROW_CALL], [n=UINT],
      [attrName=<ATTR_NAME>, attrValues=<[]ATTR_VALUE>])
 ```
 
@@ -468,7 +468,7 @@ TopN(stargazer, n=2)
 
 Filter based on an existing row:
 ```request
-TopN(Row(language=1), stargazer, n=2)
+TopN(stargazer, Row(language=1), n=2)
 ```
 ```response
 {"results":[[{"id":1240,"count":35},{"id":7508,"count":32}]]}

docs/tutorials.md

@@ -25,7 +25,7 @@ Some of our tutorials work better as standalone repos, since you can <code>git c
 
 #### Introduction
 
-Pilosa supports encrypting the communication between and to nodes in a cluster using TLS. In this tutorial, we will be setting up a three node Pilosa cluster running on the same computer. The same steps can be used for a multi-computer cluster but that requires setting up firewalls and other platform-specific configuration which is out of the scope of this tutorial.
+Pilosa supports encrypting all communication with nodes in a cluster using TLS. In this tutorial, we will be setting up a three node Pilosa cluster running on the same computer. The same steps can be used for a multi-computer cluster but that requires setting up firewalls and other platform-specific configuration which is beyond the scope of this tutorial.
 
 This tutorial assumes that you are using a UNIX-like system, such as Linux or MacOS. [Windows Subsystem for Linux (WSL)](https://msdn.microsoft.com/en-us/commandline/wsl/about) works equally well on Windows 10 systems.
 
@@ -34,11 +34,41 @@ This tutorial assumes that you are using a UNIX-like system, such as Linux or Ma
 If you haven't already done so, install Pilosa server on your computer. For Linux and WSL (Windows Subsystem for Linux) use the [Installing on Linux](../installation/#installing-on-linux) instructions. For MacOS use the [Installing on MacOS](../installation/#installing-on-macos). We do not support precompiled releases for other platforms, but you can always compile it yourself from source. See [Build from Source](../installation/#build-from-source).
 
 After installing Pilosa, you may have to add it to your `$PATH`. Check that you can run Pilosa from the command line:
-```
+``` request
 pilosa --help
 ```
+``` response
+Pilosa is a fast index to turbocharge your database.
 
-Let's create a directory for the tutorial to put all of our files and switch to that directory:
+This binary contains Pilosa itself, as well as common
+tools for administering pilosa, importing/exporting data,
+backing up, and more. Complete documentation is available
+at https://www.pilosa.com/docs/.
+
+Version: v1.0.0
+Build Time: 2018-05-14T22:14:01+0000
+
+Usage:
+  pilosa [command]
+
+Available Commands:
+  check           Do a consistency check on a pilosa data file.
+  config          Print the current configuration.
+  export          Export data from pilosa.
+  generate-config Print the default configuration.
+  help            Help about any command
+  import          Bulk load data into pilosa.
+  inspect         Get stats on a pilosa data file.
+  server          Run Pilosa.
+
+Flags:
+  -c, --config string   Configuration file to read from.
+  -h, --help            help for pilosa
+
+Use "pilosa [command] --help" for more information about a command.
+```
+
+First, create a directory in which to put all of the files for this tutorial. Then switch to that directory:
 ```
 mkdir $HOME/pilosa-tls-tutorial && cd $_
 ```
@@ -47,9 +77,9 @@ mkdir $HOME/pilosa-tls-tutorial && cd $_
 
 Securing a Pilosa cluster consists of securing the communication between nodes using TLS and Gossip encryption. [Pilosa Enterprise](https://www.pilosa.com/enterprise/) additionally supports authentication and other security features, but those are not covered in this tutorial.
 
-The first step is acquiring an SSL certificate. You can buy a commercial certificate or retrieve a Let's Encrypt certificate but we will be using a self signed certificate for practical reasons. Using self-signed certificates is not recommended in production, since it makes man in the middle attacks easy.
+The first step is acquiring an SSL certificate. You can buy a commercial certificate or retrieve a [Let's Encrypt](https://letsencrypt.org/) certificate, but we will be using a self signed certificate for practical reasons. Using self-signed certificates is not recommended in production since it makes man-in-the-middle attacks easy.
 
-The following command creates a 2048bit self-signed wildcard certificate for `*.pilosa.local` which expires 10 years later.
+The following command creates a 2048-bit, self-signed wildcard certificate for `*.pilosa.local` which expires 10 years later.
 
 ```
 openssl req -x509 -newkey rsa:2048 -keyout pilosa.local.key -out pilosa.local.crt -days 3650 -nodes -subj "/C=US/ST=Texas/L=Austin/O=Pilosa/OU=Com/CN=*.pilosa.local"
@@ -60,16 +90,16 @@ The command above creates two files in the current directory:
 * `pilosa.local.crt` is the SSL certificate.
 * `pilosa.local.key` is the private key file which must be kept as secret.
 
-Having created the SSL certificate, we can now create the gossip encryption key. Gossip encryption key file must be exactly 16, 24, or 32 bytes to select one of AES-128, AES-192, or AES-256 encryption. Reading random bytes from cryptographically secure `/dev/random` serves our purpose very well:
+Having created the SSL certificate, we can now create the gossip encryption key. The gossip encryption key file must be exactly 16, 24, or 32 bytes to select one of AES-128, AES-192, or AES-256 encryption. Reading random bytes from cryptographically secure `/dev/random` serves our purpose very well:
 ```
 head -c 32 /dev/random > pilosa.local.gossip32
 ```
 
-We now should have `pilosa.local.gossip32` in the current directory with 32 random bytes.
+We now have a file called `pilosa.local.gossip32` in the current directory which contains 32 random bytes.
 
 #### Creating the Configuration Files
 
-Pilosa supports passing configuration items using the command line, environment variables or a configuration file. We will use the last option in this tutorial and create three configuration files for our three nodes.
+Pilosa supports passing configuration items using command line options, environment variables, or a configuration file. For this tutorial, we will use three configuration files; one configuration file for each of our three nodes.
 
 One of the nodes in the cluster must be chosen as the *coordinator*. We choose the first node as the coordinator in this tutorial. The coordinator is only important during cluster resizing operations, and otherwise acts like any other node in the cluster. In the future, the coordinator will be chosen transparently by distributed consensus, and this option will be deprecated.
 
@@ -139,7 +169,7 @@ Here is some explanation of the configuration items:
 * `bind` is the address to which the server listens for incoming requests. The address is composed of three parts: scheme, host, and port. The default scheme is `http` so we explicitly specify `https` to use the HTTPS protocol for communication between nodes.
 * `[cluster]` section contains the settings for a cluster. We set `coordinator = true` for only the first node to choose that as the coordinator node. See [Cluster Configuration](../configuration/#cluster-coordinator) for other settings.
 * `[tls]` section contains the TLS settings, including the path to the SSL certificate and the corresponding key. Set `skip-verify` to `true` in order to disable host name verification and other security measures. Do not set `skip-verify` to `true` on production servers.
-* `[gossip]` section contains settings for the Gossip protocol. `seeds` contain the seed nodes which other nodes gather cluster topology. There must be at least one gossip seed. The `port` setting is the gossip listen address for the node. It should be different for each node, if the cluster is running on the same computer, otherwise you can set it to the same value. Finally, the `key` points to the gossip encryption key we created before.
+* `[gossip]` section contains settings for the gossip protocol. `seeds` contains the list of nodes from which to seed cluster membership. There must be at least one gossip seed. The `port` setting is the gossip listen address for the node. If all nodes of the cluster are running on the same computer, the gossip listen address should be different for each node. Otherwise, it can be set to the same value. Finally, the `key` points to the gossip encryption key we created earlier.
 
 #### Final Touches Before Running the Cluster
 
@@ -159,7 +189,7 @@ If any of the commands above return `ping: unknown host`, make sure your `/etc/h
 
 #### Running the Cluster
 
-Let's open three terminal windows and run each node in its window. This will enable us to better observe what's happening on which node.
+Let's open three terminal windows and run each node in its own window. This will enable us to better observe what's happening on each node.
 
 Switch to the first terminal window, change to the project directory and start the first node:
 ```
@@ -180,59 +210,72 @@ pilosa server -c node3.config.toml
 ```
 
 Let's ensure that all three Pilosa servers are running and they are connected:
-```
+``` request
 curl -k --ipv4 https://01.pilosa.local:10501/status
 ```
-
-The `-k` flag is used to tell curl that it shouldn't bother with checking the certificate the server provides and `--ipv4` workarounds an issue on MacOS where the curl requests take a long time if the address resolves to `127.0.0.1`. You can leave it out on Linux and WSL.
-
-All nodes should be in the `NORMAL` state:
 ``` response
 {"state":"NORMAL","nodes":[{"id":"98ebd177-c082-4c54-8d48-7e7c75857b52","uri":{"scheme":"https","host":"02.pilosa.local","port":10502},"isCoordinator":false},{"id":"a33dc0d6-c35f-4559-984a-e582bf032a21","uri":{"scheme":"https","host":"03.pilosa.local","port":10503},"isCoordinator":false},{"id":"e24ac014-ee2f-4cb0-b565-74df6c551f0a","uri":{"scheme":"https","host":"01.pilosa.local","port":10501},"isCoordinator":true}]}
 ```
 
+The `-k` flag is used to tell curl that it shouldn't bother checking the certificate the server provides, and the `--ipv4` flag avoids an issue on MacOS where the curl request takes a long time if the address resolves to `127.0.0.1`. You can leave it out on Linux and WSL.
+
+If everything is set up correctly, the cluster state should be `NORMAL`.
+
 #### Running Queries
 
-Having confirmed that our cluster is running OK, let's run a few queries. But before that, we need to create an index and a frame:
+Having confirmed that our cluster is running normally, let's perform a few queries. First, we need to create an index and a field:
 ``` request
-curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index -d ''
+curl https://01.pilosa.local:10501/index/sample-index \
+     -k --ipv4 \
+     -X POST
 ```
 ``` response
-{}
+{"success":true}
 ```
 
-This will create index `sample-index` with default options. Let's create the frame now:
+This will create index `sample-index` with default options. Let's create the field now:
 ``` request
-curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/frame/sample-frame -d ''
+curl https://01.pilosa.local:10501/index/sample-index/field/sample-field \
+     -k --ipv4 \
+     -X POST
 ```
 ``` response
-{}
+{"success":true}
 ```
 
-We just created frame `sample-frame` with default options.
+We just created field `sample-field` with default options.
 
-Let's run a `SetBit` query:
+Let's run a `Set` query:
 ``` request
-curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'SetBit(frame="sample-frame", row=1, col=100)'
+curl https://01.pilosa.local:10501/index/sample-index/query \
+     -k --ipv4 \
+     -X POST \
+     -d 'Set(100, sample-field=1)'
 ```
 ``` response
 {"results":[true]}
 ```
 
-Confirm that the bit was indeed set:
+Confirm that the value was indeed set:
 ``` request
-curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'Bitmap(frame="sample-frame", row=1)'
+curl https://01.pilosa.local:10501/index/sample-index/query \
+     -k --ipv4 \
+     -X POST \
+     -d 'Row(sample-field=1)'
 ```
 ``` response
-{"results":[{"attrs":{},"bits":[100]}]}
+{"results":[{"attrs":{},"columns":[100]}]}
 ```
 
 The same response should be returned when querying other nodes in the cluster:
 ``` request
-curl -k --ipv4 https://02.pilosa.local:10502/index/sample-index/query -d 'Bitmap(frame="sample-frame", row=1)'
+curl https://02.pilosa.local:10501/index/sample-index/query \
+     -k --ipv4 \
+     -X POST \
+     -d 'Row(sample-field=1)'
 ```
 ``` response
-{"results":[{"attrs":{},"bits":[100]}]}
+{"results":[{"attrs":{},"columns":[100]}]}
 ```
 
 #### What's Next?
@@ -265,7 +308,7 @@ docker run -it --rm --name pilosa1 -p 10101:10101 --network=pilosanet pilosa/pil
 
 Let's run the second Pilosa node and attach it to the virtual network as well. Note that we set the address of the gossip seed to the address of the first node:
 ```
-docker run -it --rm --name pilosa2 --network=pilosanet pilosa/pilosa:latest server --bind pilosa2 --gossip.seeds=pilosa1:14000
+docker run -it --rm --name pilosa2 -p 10102:10101 --network=pilosanet pilosa/pilosa:latest server --bind pilosa2 --gossip.seeds=pilosa1:14000
 ```
 
 Let's test that the nodes in the cluster connected with each other:
@@ -273,7 +316,7 @@ Let's test that the nodes in the cluster connected with each other:
 curl localhost:10101/status
 ```
 ``` response
-{"state":"NORMAL","nodes":[{"id":"2e8332d0-1fee-44dd-a359-e0d6ecbcefc1","uri":{"scheme":"http","host":"pilosa1","port":10101},"isCoordinator":true},{"id":"8c0dbcdc-9503-4265-8ad2-ba85a4bb10fa","uri":{"scheme":"http","host":"pilosa2","port":10101},"isCoordinator":false}]}
+{"state":"NORMAL","nodes":[{"id":"2e8332d0-1fee-44dd-a359-e0d6ecbcefc1","uri":{"scheme":"http","host":"pilosa1","port":10101},"isCoordinator":true},{"id":"8c0dbcdc-9503-4265-8ad2-ba85a4bb10fa","uri":{"scheme":"http","host":"pilosa2","port":10101},"isCoordinator":false}],"localID":"2e8332d0-1fee-44dd-a359-e0d6ecbcefc1"}
 ```
 
 And similarly for the second node:
@@ -281,7 +324,7 @@ And similarly for the second node:
 curl localhost:10102/status
 ```
 ``` response
-{"state":"NORMAL","nodes":[{"id":"2e8332d0-1fee-44dd-a359-e0d6ecbcefc1","uri":{"scheme":"http","host":"pilosa1","port":10101},"isCoordinator":true},{"id":"8c0dbcdc-9503-4265-8ad2-ba85a4bb10fa","uri":{"scheme":"http","host":"pilosa2","port":10101},"isCoordinator":false}]}
+{"state":"NORMAL","nodes":[{"id":"2e8332d0-1fee-44dd-a359-e0d6ecbcefc1","uri":{"scheme":"http","host":"pilosa1","port":10101},"isCoordinator":true},{"id":"8c0dbcdc-9503-4265-8ad2-ba85a4bb10fa","uri":{"scheme":"http","host":"pilosa2","port":10101},"isCoordinator":false}],"localID":"2e8332d0-1fee-44dd-a359-e0d6ecbcefc1"}
 ```
 The corresponding [Docker Compose](https://docs.docker.com/compose/) file is below:
 
@@ -304,6 +347,8 @@ services:
       - "pilosa1:10101"
   pilosa2:
     image: pilosa/pilosa:latest
+    ports:
+      - "10102:10101"
     environment:
       - PILOSA_GOSSIP_SEEDS=pilosa1:14000
     networks:
@@ -319,13 +364,13 @@ networks:
 
 #### Running a Docker Swarm
 
-It is very easy to run a Pilosa Cluster on different servers using [Docker Swarm mode](https://docs.docker.com/engine/swarm/). All we have to do is creating an overlay network instead of the bridge network.
+It is very easy to run a Pilosa Cluster on different servers using [Docker Swarm mode](https://docs.docker.com/engine/swarm/). All we have to do is create an overlay network instead of a bridge network.
 
-The instructions in this section require Docker 17.06 and better. Although it is possible to run a Docker swarm on MacOS or Windows, it is easiest to run it on Linux. So we assume you are trying these instructions on Linux, probably on the cloud.
+The instructions in this section require Docker 17.06 or newer. Although it is possible to run a Docker swarm on MacOS or Windows, it is easiest to run it on Linux. The following instructions assume you are running on Linux.
 
 We are going to use two servers: the manager node runs in the first server and a worker node in the second server.
 
-Docker nodes require some ports to be accesible from outside. Before carrying on, make sure the following ports are open on all nodes: TCP/2377, TCP/7946, UDP/7946, UDP/4789.
+Docker nodes require some ports to be accesible from the outside. Before proceeding, make sure the following ports are open on all nodes: TCP/2377, TCP/7946, UDP/7946, UDP/4789.
 
 Let's initialize the swarm first. Run the following on the manager:
 ```
@@ -383,10 +428,10 @@ These were the same commands we used in the previous section except the port map
 docker run -it --rm --network=pilosanet --name shell alpine wget -q -O- pilosa1:10101/status
 ```
 ``` response
-{"state":"NORMAL","nodes":[{"id":"3e3b0abd-1945-441a-a01f-5a28272972f5","uri":{"scheme":"http","host":"pilosa1","port":10101},"isCoordinator":true},{"id":"71ed27cc-9443-4f41-88fb-1c22f92bf695","uri":{"scheme":"http","host":"pilosa2","port":10101},"isCoordinator":false}]}
+{"state":"NORMAL","nodes":[{"id":"3e3b0abd-1945-441a-a01f-5a28272972f5","uri":{"scheme":"http","host":"pilosa1","port":10101},"isCoordinator":true},{"id":"71ed27cc-9443-4f41-88fb-1c22f92bf695","uri":{"scheme":"http","host":"pilosa2","port":10101},"isCoordinator":false}],"localID":"3e3b0abd-1945-441a-a01f-5a28272972f5"}
 ```
 
-You can add as many as worker nodes to both the swarm and the Pilosa cluster using the steps above.
+You can add additional worker nodes to both the swarm and the Pilosa cluster using the steps above.
 
 #### What's Next?
 
@@ -410,7 +455,7 @@ curl localhost:10101/index/patients \
 {"success":true}
 ```
 
-In addition to storing rows of bits, a frame can also contain fields that store integer values. The next steps creates three fields (`age`, `weight`, `tcells`) in the `measurements` frame.
+In addition to storing rows of bits, a field can also store integer values. The next steps creates three fields (`age`, `weight`, `tcells`) in the `measurements` field.
 ``` request
 curl localhost:10101/index/patients/field/age \
      -X POST \

executor.go

@@ -101,9 +101,12 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar
 	}
 
 	// Translate query keys to ids, if necessary.
-	for i := range q.Calls {
-		if err := e.translateCall(index, idx, q.Calls[i]); err != nil {
-			return nil, err
+	// No need to translate a remote call.
+	if !opt.Remote {
+		for i := range q.Calls {
+			if err := e.translateCall(index, idx, q.Calls[i]); err != nil {
+				return nil, err
+			}
 		}
 	}
 
@@ -113,10 +116,13 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar
 	}
 
 	// Translate response objects from ids to keys, if necessary.
-	for i := range results {
-		results[i], err = e.translateResult(index, idx, q.Calls[i], results[i])
-		if err != nil {
-			return nil, err
+	// No need to translate a remote call.
+	if !opt.Remote {
+		for i := range results {
+			results[i], err = e.translateResult(index, idx, q.Calls[i], results[i])
+			if err != nil {
+				return nil, err
+			}
 		}
 	}
 	return results, nil
@@ -712,7 +718,7 @@ func (e *executor) executeIntersectShard(ctx context.Context, index string, c *p
 		if i == 0 {
 			other = row
 		} else {
-			other = other.intersect(row)
+			other = other.Intersect(row)
 		}
 	}
 	other.invalidateCount()
@@ -1256,7 +1262,7 @@ func (e *executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal
 
 		rowID, ok, err := c.UintArg("_" + rowLabel)
 		if err != nil {
-			return nil, fmt.Errorf("reading SetRowAttrs() row: %v", rowLabel)
+			return nil, errors.Wrap(err, "reading SetRowAttrs() row")
 		} else if !ok {
 			return nil, fmt.Errorf("SetRowAttrs row field '%v' required", rowLabel)
 		}
@@ -1550,6 +1556,10 @@ func (e *executor) translateCall(index string, idx *Index, c *pql.Call) error {
 		colKey = "_" + columnLabel
 		fieldName, _ = c.FieldArg()
 		rowKey = fieldName
+	} else if c.Name == "SetRowAttrs" {
+		// Positional args in new PQL syntax require special handling here.
+		rowKey = "_" + rowLabel
+		fieldName = callArgString(c, "_field")
 	} else {
 		colKey = "col"
 		fieldName = callArgString(c, "field")

executor_test.go

@@ -471,29 +471,53 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) {
 		t.Fatal(err)
 	} else if _, err := index.CreateFieldIfNotExists("xxx", pilosa.OptFieldTypeDefault()); err != nil {
 		t.Fatal(err)
-	}
-
-	// Set two attrs on f/10.
-	// Also set attrs on other bitmaps and fields to test isolation.
-	if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(f, 10, foo="bar")`}); err != nil {
-		t.Fatal(err)
-	}
-	if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(f, 200, YYY=1)`}); err != nil {
-		t.Fatal(err)
-	}
-	if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(xxx, 10, YYY=1)`}); err != nil {
-		t.Fatal(err)
-	}
-	if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(f, 10, baz=123, bat=true)`}); err != nil {
+	} else if _, err := index.CreateFieldIfNotExists("kf", pilosa.OptFieldTypeDefault(), pilosa.OptFieldKeys()); err != nil {
 		t.Fatal(err)
 	}
 
-	f := hldr.Field("i", "f")
-	if m, err := f.RowAttrStore().Attrs(10); err != nil {
-		t.Fatal(err)
-	} else if !reflect.DeepEqual(m, map[string]interface{}{"foo": "bar", "baz": int64(123), "bat": true}) {
-		t.Fatalf("unexpected bitmap attr: %#v", m)
-	}
+	t.Run("rowID", func(t *testing.T) {
+		// Set two attrs on f/10.
+		// Also set attrs on other bitmaps and fields to test isolation.
+		if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(f, 10, foo="bar")`}); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(f, 200, YYY=1)`}); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(xxx, 10, YYY=1)`}); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(f, 10, baz=123, bat=true)`}); err != nil {
+			t.Fatal(err)
+		}
+
+		f := hldr.Field("i", "f")
+		if m, err := f.RowAttrStore().Attrs(10); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(m, map[string]interface{}{"foo": "bar", "baz": int64(123), "bat": true}) {
+			t.Fatalf("unexpected bitmap attr: %#v", m)
+		}
+	})
+
+	t.Run("rowKey", func(t *testing.T) {
+		// Set two attrs on f/10.
+		// Also set attrs on other bitmaps and fields to test isolation.
+		if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(kf, "row10", foo="bar")`}); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(kf, "row200", YYY=1)`}); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(kf, "row10", baz=123, bat=true)`}); err != nil {
+			t.Fatal(err)
+		}
+
+		if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(kf="row10")`}); err != nil {
+			t.Fatal(err)
+		} else if attrs := result.Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123), "bat": true}) {
+			t.Fatalf("unexpected attrs: %+v", attrs)
+		}
+	})
 }
 
 // Ensure a TopN() query can be executed.

fragment.go

@@ -598,7 +598,7 @@ func (f *fragment) min(filter *Row, bitDepth uint) (min, count uint64, err error
 
 	consider := f.row(uint64(bitDepth))
 	if filter != nil {
-		consider = consider.intersect(filter)
+		consider = consider.Intersect(filter)
 	}
 
 	// If there are no columns to consider, return early.
@@ -631,7 +631,7 @@ func (f *fragment) max(filter *Row, bitDepth uint) (max, count uint64, err error
 
 	consider := f.row(uint64(bitDepth))
 	if filter != nil {
-		consider = consider.intersect(filter)
+		consider = consider.Intersect(filter)
 	}
 
 	// If there are no columns to consider, return early.
@@ -643,7 +643,7 @@ func (f *fragment) max(filter *Row, bitDepth uint) (max, count uint64, err error
 		ii := i - 1 // allow for uint range: (bitDepth-1) to 0
 		row := f.row(uint64(ii))
 
-		x := row.intersect(consider)
+		x := row.Intersect(consider)
 		count = x.Count()
 		if count > 0 {
 			max += (1 << ii)
@@ -682,7 +682,7 @@ func (f *fragment) rangeEQ(bitDepth uint, predicate uint64) (*Row, error) {
 		bit := (predicate >> uint(i)) & 1
 
 		if bit == 1 {
-			b = b.intersect(row)
+			b = b.Intersect(row)
 		} else {
 			b = b.Difference(row)
 		}
@@ -783,7 +783,7 @@ func (f *fragment) rangeGT(bitDepth uint, predicate uint64, allowEquality bool)
 		// If bit is unset then add columns with set bit to keep.
 		// Don't bother to compute this on the final iteration.
 		if i > 0 {
-			keep = keep.Union(b.intersect(row))
+			keep = keep.Union(b.Intersect(row))
 		}
 	}
 
@@ -815,7 +815,7 @@ func (f *fragment) rangeBetween(bitDepth uint, predicateMin, predicateMax uint64
 			// If bit is unset then add columns with set bit to keep.
 			// Don't bother to compute this on the final iteration.
 			if i > 0 {
-				keep1 = keep1.Union(b.intersect(row))
+				keep1 = keep1.Union(b.Intersect(row))
 			}
 		}
 

holder.go

@@ -232,7 +232,7 @@ func (h *Holder) Schema() []*IndexInfo {
 func (h *Holder) limitedSchema() []*IndexInfo {
 	var a []*IndexInfo
 	for _, index := range h.Indexes() {
-		di := &IndexInfo{Name: index.Name()}
+		di := &IndexInfo{Name: index.Name(), Options: index.Options()}
 		for _, field := range index.Fields() {
 			fi := &FieldInfo{Name: field.Name(), Options: field.Options()}
 			di.Fields = append(di.Fields, fi)
@@ -248,8 +248,7 @@ func (h *Holder) limitedSchema() []*IndexInfo {
 func (h *Holder) applySchema(schema *Schema) error {
 	// Create indexes that don't exist.
 	for _, index := range schema.Indexes {
-		opt := IndexOptions{}
-		idx, err := h.CreateIndexIfNotExists(index.Name, opt)
+		idx, err := h.CreateIndexIfNotExists(index.Name, index.Options)
 		if err != nil {
 			return errors.Wrap(err, "creating index")
 		}

http/handler.go

@@ -1303,14 +1303,14 @@ func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Reques
 
 type defaultClusterMessageResponse struct{}
 
+// translateStoreBufferSize is the buffer size used for streaming data.
+const translateStoreBufferSize = 65536
+
 func (h *Handler) handleGetTranslateData(w http.ResponseWriter, r *http.Request) {
 	q := r.URL.Query()
 	offset, _ := strconv.ParseInt(q.Get("offset"), 10, 64)
 
-	pipeR, pipeW := io.Pipe()
-
-	err := h.api.GetTranslateData(r.Context(), pipeW, offset)
-
+	rdr, err := h.api.GetTranslateData(r.Context(), offset)
 	if err != nil {
 		if errors.Cause(err) == pilosa.ErrNotImplemented {
 			http.Error(w, err.Error(), http.StatusNotImplemented)
@@ -1326,7 +1326,28 @@ func (h *Handler) handleGetTranslateData(w http.ResponseWriter, r *http.Request)
 		w.Flush()
 	}
 
-	io.Copy(w, pipeR)
+	// Copy from reader to client until store or client disconnect.
+	buf := make([]byte, translateStoreBufferSize)
+	for {
+		// Read from store.
+		n, err := rdr.Read(buf)
+		if err == io.EOF {
+			return
+		} else if err != nil {
+			h.logger.Printf("http: translate store read error: %s", err)
+			return
+		} else if n == 0 {
+			continue
+		}
+
+		// Write to response & flush.
+		if _, err := w.Write(buf[:n]); err != nil {
+			h.logger.Printf("http: translate store response write error: %s", err)
+			return
+		} else if w, ok := w.(http.Flusher); ok {
+			w.Flush()
+		}
+	}
 }
 
 type queryValidationSpec struct {

http/handler_internal_test.go

@@ -31,6 +31,7 @@ func TestPostIndexRequestUnmarshalJSON(t *testing.T) {
 		err      string
 	}{
 		{json: `{"options": {}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{}}},
+		{json: `{"options": {"keys": true}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{Keys: true}}},
 		{json: `{"options": 4}`, err: "options is not map[string]interface{}"},
 		{json: `{"option": {}}`, err: "Unknown key: option:map[]"},
 		{json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"},

http/translator.go

@@ -16,7 +16,7 @@ import (
 // Ensure implementation implements inteface.
 var _ pilosa.TranslateStore = (*translateStore)(nil)
 
-// translateStore represents an implementation of translateStore that
+// translateStore represents an implementation of pilosa.TranslateStore that
 // communicates over HTTP. This is used with the TranslateHandler.
 type translateStore struct {
 	URL string

http/translator_test.go

@@ -2,9 +2,9 @@ package http_test
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"io/ioutil"
-	gohttp "net/http"
 	"testing"
 	"time"
 
@@ -30,74 +30,53 @@ func TestTranslateStore_Reader(t *testing.T) {
 	// Ensure client can connect and stream the translate store data.
 	t.Run("OK", func(t *testing.T) {
 		t.Run("ServerDisconnect", func(t *testing.T) {
-			var mrc mock.ReadCloser
-			var readN int
-			mrc.ReadFunc = func(p []byte) (int, error) {
-				readN++
-				switch readN {
-				case 1:
-					copy(p, []byte("foo"))
-					return 3, nil
-				case 2:
-					copy(p, []byte("barbaz"))
-					return 6, nil
-				case 3:
-					return 0, io.EOF
-				default:
-					t.Fatal("unexpected read")
-					return 0, nil
-				}
-			}
-			closeInvoked := make(chan struct{})
-			mrc.CloseFunc = func() error {
-				close(closeInvoked)
-				return nil
+			// This test is currently flawed, breaking intermittently with message:
+			// "translator_test.go:65: unexpected EOF"
+			t.Skip()
+
+			primary := test.MustRunCluster(t, 1)[0]
+
+			hldr := test.Holder{Holder: primary.Server.Holder()}
+			index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true})
+			_, err := index.CreateField("f", pilosa.OptFieldTypeDefault())
+			if err != nil {
+				t.Fatal(err)
 			}
 
-			// Setup handler on test server.
-			var translateStore mock.TranslateStore
-
-			translateStore.ReaderFunc = func(ctx context.Context, off int64) (io.ReadCloser, error) {
-				// Check context to make sure this is the call we are looking for.
-				// (Something else calls ReaderFunc on server startup)
-				if ctx.Value(gohttp.ServerContextKey) != nil {
-					if off != 100 {
-						t.Fatalf("unexpected off: %d", off)
-					}
-					return &mrc, nil
-				}
-				return newMockReadCloser(), nil
+			// Set data on the primary node.
+			if _, err := primary.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` +
+				fmt.Sprintf("Set(%s, f=%d)\n", `"foo"`, 10) +
+				fmt.Sprintf("Set(%s, f=%d)\n", `"bar"`, 10) +
+				fmt.Sprintf("Set(%s, f=%d)\n", `"baz"`, 10),
+			}); err != nil {
+				t.Fatal(err)
 			}
 
-			opts := server.OptCommandServerOptions(pilosa.OptServerPrimaryTranslateStore(translateStore))
-			main := test.MustRunCluster(t, 1, []server.CommandOption{opts})[0]
-
-			defer main.Close()
-
 			// Connect to server and stream all available data.
-			store := http.NewTranslateStore(main.URL())
+			store := http.NewTranslateStore(primary.URL())
+
+			// Wait to ensure writes make it to translate store
+			time.Sleep(100 * time.Millisecond)
+
+			rc, err := store.Reader(context.Background(), 11) // offset=11 skips the first entry: \n\x01\x01i\x00\x01\x01\x03foo
+
+			// Close the primary to disconnect reader.
+			primary.Close()
 
-			rc, err := store.Reader(context.Background(), 100)
 			if err != nil {
 				t.Fatal(err)
 			} else if data, err := ioutil.ReadAll(rc); err != nil {
 				t.Fatal(err)
-			} else if string(data) != `foobarbaz` {
+			} else if string(data) != "\n\x01\x01i\x00\x01\x02\x03bar\n\x01\x01i\x00\x01\x03\x03baz" {
 				t.Fatalf("unexpected data: %q", data)
 			} else if err := rc.Close(); err != nil {
 				t.Fatal(err)
 			}
-
-			select {
-			case <-time.NewTimer(time.Millisecond * 100).C:
-				t.Fatal("expected server close")
-			case <-closeInvoked:
-				return
-			}
 		})
 
 		// Ensure server closes store reader if client disconnects.
 		t.Run("ClientDisconnect", func(t *testing.T) {
+			t.Skip() // can't mock server from http package
 			// Setup mock so that Read() hangs.
 			done := make(chan struct{})
 
@@ -121,14 +100,14 @@ func TestTranslateStore_Reader(t *testing.T) {
 			}
 
 			opts := server.OptCommandServerOptions(pilosa.OptServerPrimaryTranslateStore(translateStore))
-			main := test.MustRunCluster(t, 1, []server.CommandOption{opts})[0]
+			primary := test.MustRunCluster(t, 1, []server.CommandOption{opts})[0]
 
-			defer main.Close()
+			defer primary.Close()
 			defer close(done)
 
 			// Connect to server and begin streaming.
 			ctx, cancel := context.WithCancel(context.Background())
-			store := http.NewTranslateStore(main.URL())
+			store := http.NewTranslateStore(primary.URL())
 			if _, err := store.Reader(ctx, 0); err != nil {
 				t.Fatal(err)
 			}
@@ -146,16 +125,17 @@ func TestTranslateStore_Reader(t *testing.T) {
 
 	// Ensure client is notified if the server doesn't support streaming replication.
 	t.Run("ErrNotImplemented", func(t *testing.T) {
+		t.Skip() // can't mock server from http package
 		var translateStore mock.TranslateStore
 		translateStore.ReaderFunc = func(ctx context.Context, off int64) (io.ReadCloser, error) {
 			return nil, pilosa.ErrNotImplemented
 		}
 
 		opts := server.OptCommandServerOptions(pilosa.OptServerPrimaryTranslateStore(translateStore))
-		main := test.MustRunCluster(t, 1, []server.CommandOption{opts})[0]
-		defer main.Close()
+		primary := test.MustRunCluster(t, 1, []server.CommandOption{opts})[0]
+		defer primary.Close()
 
-		_, err := http.NewTranslateStore(main.URL()).Reader(context.Background(), 0)
+		_, err := http.NewTranslateStore(primary.URL()).Reader(context.Background(), 0)
 		if err != pilosa.ErrNotImplemented {
 			t.Fatalf("unexpected error: %s", err)
 		}

index.go

@@ -95,7 +95,7 @@ func (i *Index) Options() IndexOptions {
 }
 
 func (i *Index) options() IndexOptions {
-	return IndexOptions{}
+	return IndexOptions{Keys: i.keys}
 }
 
 // Open opens and initializes the index.
@@ -292,7 +292,7 @@ func (i *Index) CreateField(name string, opts ...FieldOption) (*Field, error) {
 }
 
 // CreateFieldIfNotExists creates a field with the given options if it doesn't exist.
-func (i *Index) CreateFieldIfNotExists(name string, opts FieldOption) (*Field, error) {
+func (i *Index) CreateFieldIfNotExists(name string, opts ...FieldOption) (*Field, error) {
 	i.mu.Lock()
 	defer i.mu.Unlock()
 
@@ -301,11 +301,13 @@ func (i *Index) CreateFieldIfNotExists(name string, opts FieldOption) (*Field, e
 		return f, nil
 	}
 
-	// Apply functional option.
+	// Apply functional options.
 	fo := FieldOptions{}
-	err := opts(&fo)
-	if err != nil {
-		return nil, errors.Wrap(err, "applying option")
+	for _, opt := range opts {
+		err := opt(&fo)
+		if err != nil {
+			return nil, errors.Wrap(err, "applying option")
+		}
 	}
 
 	return i.createField(name, fo)
@@ -406,7 +408,7 @@ func (p indexSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() }
 // IndexInfo represents schema information for an index.
 type IndexInfo struct {
 	Name    string       `json:"name"`
-	options IndexOptions `json:"options"`
+	Options IndexOptions `json:"options"`
 	Fields  []*FieldInfo `json:"fields"`
 }
 

mock/translator.go

@@ -12,8 +12,8 @@ var _ pilosa.TranslateStore = (*TranslateStore)(nil)
 type TranslateStore struct {
 	TranslateColumnsToUint64Func func(index string, values []string) ([]uint64, error)
 	TranslateColumnToStringFunc  func(index string, values uint64) (string, error)
-	TranslateRowsToUint64Func    func(index, frame string, values []string) ([]uint64, error)
-	TranslateRowToStringFunc     func(index, frame string, values uint64) (string, error)
+	TranslateRowsToUint64Func    func(index, field string, values []string) ([]uint64, error)
+	TranslateRowToStringFunc     func(index, field string, values uint64) (string, error)
 	ReaderFunc                   func(ctx context.Context, off int64) (io.ReadCloser, error)
 }
 
@@ -25,12 +25,12 @@ func (s TranslateStore) TranslateColumnToString(index string, values uint64) (st
 	return s.TranslateColumnToStringFunc(index, values)
 }
 
-func (s TranslateStore) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) {
-	return s.TranslateRowsToUint64Func(index, frame, values)
+func (s TranslateStore) TranslateRowsToUint64(index, field string, values []string) ([]uint64, error) {
+	return s.TranslateRowsToUint64Func(index, field, values)
 }
 
-func (s TranslateStore) TranslateRowToString(index, frame string, value uint64) (string, error) {
-	return s.TranslateRowToStringFunc(index, frame, value)
+func (s TranslateStore) TranslateRowToString(index, field string, value uint64) (string, error) {
+	return s.TranslateRowToStringFunc(index, field, value)
 }
 
 func (s TranslateStore) Reader(ctx context.Context, off int64) (io.ReadCloser, error) {

pql/pql.peg

@@ -7,7 +7,7 @@ type PQL Peg {
 
 Calls <- sp (Call sp)* !.
 Call <-  'Set' {p.startCall("Set")} open col comma args (comma timestamp)? close {p.endCall()}
-       / 'SetRowAttrs' {p.startCall("SetRowAttrs")} open posfield comma uintrow comma args close {p.endCall()}
+       / 'SetRowAttrs' {p.startCall("SetRowAttrs")} open posfield comma row comma args close {p.endCall()}
        / 'SetColumnAttrs' {p.startCall("SetColumnAttrs")} open col comma args close {p.endCall()}
        / 'Clear' {p.startCall("Clear")} open col comma args close {p.endCall()}
        / 'TopN' {p.startCall("TopN")} open posfield (comma allargs)? close {p.endCall()}
@@ -60,6 +60,10 @@ col <- ( <uint> {p.addPosNum("_col", buffer[begin:end])}
         / '\'' <singlequotedstring> '\'' {p.addPosStr("_col", buffer[begin:end])}
         / '"' <doublequotedstring> '"' {p.addPosStr("_col", buffer[begin:end])}
         )
+row <- ( <uint> {p.addPosNum("_row", buffer[begin:end])}
+        / '\'' <singlequotedstring> '\'' {p.addPosStr("_row", buffer[begin:end])}
+        / '"' <doublequotedstring> '"' {p.addPosStr("_row", buffer[begin:end])}
+        )
 
 open <- '(' sp
 close <- ')' sp

pql/pqlpeg_test.go

@@ -160,6 +160,14 @@ func TestPEGWorking(t *testing.T) {
 			name:   "SetRowAttrs2args",
 			input:  "SetRowAttrs(blah, 9, a=47, b=bval)",
 			ncalls: 1},
+		{
+			name:   "SetRowAttrsWithRowKeySingleQuote",
+			input:  "SetRowAttrs(blah, 'rowKey', a=47)",
+			ncalls: 1},
+		{
+			name:   "SetRowAttrsWithRowKeyDoubleQuote",
+			input:  `SetRowAttrs(blah, "rowKey", a=47)`,
+			ncalls: 1},
 		{
 			name:   "SetColumnAttrs",
 			input:  "SetColumnAttrs(9, a=47)",
@@ -168,6 +176,14 @@ func TestPEGWorking(t *testing.T) {
 			name:   "SetColumnAttrs2args",
 			input:  "SetColumnAttrs(9, a=47, b=bval)",
 			ncalls: 1},
+		{
+			name:   "SetColumnAttrsWithColKeySingleQuote",
+			input:  "SetColumnAttrs('colKey', a=47)",
+			ncalls: 1},
+		{
+			name:   "SetColumnAttrsWithColKeyDoubleQuote",
+			input:  `SetColumnAttrs("colKey", a=47)`,
+			ncalls: 1},
 		{
 			name:   "Clear",
 			input:  "Clear(1, a=53)",
@@ -330,6 +346,28 @@ func TestPQLDeepEquality(t *testing.T) {
 					"_row":   int64(9),
 				},
 			}},
+		{
+			name: "SetRowAttrsWithRowKeySingleQuote",
+			call: "SetRowAttrs(myfield, 'rowKey', z=4)",
+			exp: &Call{
+				Name: "SetRowAttrs",
+				Args: map[string]interface{}{
+					"z":      int64(4),
+					"_field": "myfield",
+					"_row":   "rowKey",
+				},
+			}},
+		{
+			name: "SetRowAttrsWithRowKeyDoubleQuote",
+			call: `SetRowAttrs(myfield, "rowKey", z=4)`,
+			exp: &Call{
+				Name: "SetRowAttrs",
+				Args: map[string]interface{}{
+					"z":      int64(4),
+					"_field": "myfield",
+					"_row":   "rowKey",
+				},
+			}},
 		{
 			name: "SetColumnAttrs",
 			call: "SetColumnAttrs(9, z=4)",
@@ -340,6 +378,26 @@ func TestPQLDeepEquality(t *testing.T) {
 					"_col": int64(9),
 				},
 			}},
+		{
+			name: "SetColumnAttrsWithColKeySingleQuote",
+			call: "SetColumnAttrs('colKey', z=4)",
+			exp: &Call{
+				Name: "SetColumnAttrs",
+				Args: map[string]interface{}{
+					"z":    int64(4),
+					"_col": "colKey",
+				},
+			}},
+		{
+			name: "SetColumnAttrsWithColKeyDoubleQuote",
+			call: `SetColumnAttrs("colKey", z=4)`,
+			exp: &Call{
+				Name: "SetColumnAttrs",
+				Args: map[string]interface{}{
+					"z":    int64(4),
+					"_col": "colKey",
+				},
+			}},
 		{
 			name: "Clear",
 			call: "Clear(1, a=7)",

row.go

@@ -82,8 +82,8 @@ func (r *Row) intersectionCount(other *Row) uint64 {
 	return n
 }
 
-// intersect returns the itersection of r and other.
-func (r *Row) intersect(other *Row) *Row {
+// Intersect returns the itersection of r and other.
+func (r *Row) Intersect(other *Row) *Row {
 	var segments []rowSegment
 
 	itr := newMergeSegmentIterator(r.segments, other.segments)

server.go

@@ -457,8 +457,8 @@ func (s *Server) receiveMessage(m Message) error {
 		}
 		idx.setRemoteMaxShard(obj.Shard)
 	case *CreateIndexMessage:
-		opt := IndexOptions{}
-		_, err := s.holder.CreateIndex(obj.Index, opt)
+		opt := obj.Meta
+		_, err := s.holder.CreateIndex(obj.Index, *opt)
 		if err != nil {
 			return err
 		}

server/config.go

@@ -74,7 +74,7 @@ type Config struct {
 	// Translation config supports translation store replication.
 	Translation struct {
 		PrimaryURL string `toml:"primary-url"`
-	}
+	} `toml:"translation"`
 
 	AntiEntropy struct {
 		Interval toml.Duration `toml:"interval"`

server/handler_test.go

@@ -24,6 +24,7 @@ import (
 	"reflect"
 	"strings"
 	"testing"
+	"time"
 
 	gohttp "net/http"
 
@@ -655,6 +656,56 @@ func TestHandler_Endpoints(t *testing.T) {
 	})
 }
 
+func TestClusterTranslator(t *testing.T) {
+	cluster := make(test.Cluster, 2)
+	cluster[0] = test.NewCommandNode(true)
+	cluster[0].Config.Gossip.Port = "0"
+	cluster[0].Start()
+	httpTranslateStore := http.NewTranslateStore(cluster[0].URL())
+	cluster[1] = test.NewCommandNode(false,
+		server.OptCommandServerOptions(
+			pilosa.OptServerPrimaryTranslateStore(httpTranslateStore),
+		),
+	)
+	cluster[1].Config.Gossip.Port = "0"
+	cluster[1].Config.Gossip.Seeds = []string{cluster[0].GossipAddress()}
+	cluster[1].Start()
+
+	test.MustDo("POST", cluster[0].URL()+"/index/i0", "{\"options\": {\"keys\": true}}")
+	test.MustDo("POST", cluster[0].URL()+"/index/i0/field/f0", "{\"options\": {\"keys\": true}}")
+
+	test.MustDo("POST", cluster[0].URL()+"/index/i0/query", "Set(\"foo\", f0=\"bar\")")
+
+	// wait for key to replicate to second node
+	time.Sleep(100 * time.Millisecond)
+
+	result0 := test.MustDo("POST", cluster[0].URL()+"/index/i0/query", "Row(f0=\"bar\")").Body
+	result1 := test.MustDo("POST", cluster[1].URL()+"/index/i0/query", "Row(f0=\"bar\")").Body
+
+	if result0 != result1 {
+		t.Fatalf("`%s` != `%s`", result0, result1)
+	}
+
+	for _, i := range []string{result0, result1} {
+		var resp map[string]interface{}
+		err := json.Unmarshal([]byte(i), &resp)
+		if err != nil {
+			t.Fatalf("json unmarshal error: %s", err)
+		}
+		if results, ok := resp["results"].([]interface{}); ok {
+			if result, ok := results[0].(map[string]interface{}); ok {
+				if keys, ok := result["keys"].([]interface{}); ok {
+					if key, ok := keys[0].(string); ok {
+						if key != "foo" {
+							t.Fatalf("Key is %s but should be 'foo'", key)
+						}
+					}
+				}
+			}
+		}
+	}
+}
+
 func mustJSONDecode(t *testing.T, r io.Reader) (ret map[string]interface{}) {
 	dec := json.NewDecoder(r)
 	err := dec.Decode(&ret)

test/index.go

@@ -83,8 +83,8 @@ func (i *Index) CreateField(name string, opts ...pilosa.FieldOption) (*Field, er
 }
 
 // CreateFieldIfNotExists creates a field with the given options if it doesn't exist.
-func (i *Index) CreateFieldIfNotExists(name string, opts pilosa.FieldOption) (*Field, error) {
-	f, err := i.Index.CreateFieldIfNotExists(name, opts)
+func (i *Index) CreateFieldIfNotExists(name string, opts ...pilosa.FieldOption) (*Field, error) {
+	f, err := i.Index.CreateFieldIfNotExists(name, opts...)
 	if err != nil {
 		return nil, err
 	}

translate.go

@@ -31,7 +31,7 @@ var (
 	ErrTranslateStoreClosed       = errors.New("pilosa: translate store closed")
 	ErrTranslateStoreReaderClosed = errors.New("pilosa: translate store reader closed")
 	ErrReplicationNotSupported    = errors.New("pilosa: replication not supported")
-	ErrTranslateStoreReadOnly     = errors.New("pilosa: operation not supported, translate store read only")
+	ErrTranslateStoreReadOnly     = errors.New("pilosa: translate store could not find or create key, translate store read only")
 )
 
 // TranslateStore is the storage for translation string-to-uint64 values.
@@ -39,8 +39,8 @@ type TranslateStore interface {
 	TranslateColumnsToUint64(index string, values []string) ([]uint64, error)
 	TranslateColumnToString(index string, values uint64) (string, error)
 
-	TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error)
-	TranslateRowToString(index, frame string, values uint64) (string, error)
+	TranslateRowsToUint64(index, field string, values []string) ([]uint64, error)
+	TranslateRowToString(index, field string, values uint64) (string, error)
 
 	// Returns a reader from the given offset of the raw data file.
 	// The returned reader must be closed by the caller when done.
@@ -64,7 +64,7 @@ type TranslateFile struct {
 	closing chan struct{}
 
 	cols map[string]*index
-	rows map[frameKey]*index
+	rows map[fieldKey]*index
 
 	Path    string
 	mapSize int
@@ -82,7 +82,7 @@ func NewTranslateFile() *TranslateFile {
 		writeNotify: make(chan struct{}),
 		closing:     make(chan struct{}),
 		cols:        make(map[string]*index),
-		rows:        make(map[frameKey]*index),
+		rows:        make(map[fieldKey]*index),
 
 		mapSize: defaultMapSize,
 
@@ -201,7 +201,7 @@ func (s *TranslateFile) applyEntry(entry *LogEntry, offset int64) error {
 		idx = s.col(string(entry.Index))
 
 	case LogEntryTypeInsertRow:
-		idx = s.row(string(entry.Index), string(entry.Frame))
+		idx = s.row(string(entry.Index), string(entry.Field))
 
 	default:
 		return fmt.Errorf("enterprise.TranslateFile.applyEntry(): unknown log entry type: 0x%20x", entry.Type)
@@ -318,11 +318,11 @@ func (s *TranslateFile) col(index string) *index {
 	return idx
 }
 
-func (s *TranslateFile) row(index, frame string) *index {
-	idx := s.rows[frameKey{index, frame}]
+func (s *TranslateFile) row(index, field string) *index {
+	idx := s.rows[fieldKey{index, field}]
 	if idx == nil {
 		idx = newIndex(s.data)
-		s.rows[frameKey{index, frame}] = idx
+		s.rows[fieldKey{index, field}] = idx
 	}
 	return idx
 }
@@ -433,8 +433,8 @@ func (s *TranslateFile) TranslateColumnToString(index string, value uint64) (str
 	return "", nil
 }
 
-func (s *TranslateFile) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) {
-	key := frameKey{index, frame}
+func (s *TranslateFile) TranslateRowsToUint64(index, field string, values []string) ([]uint64, error) {
+	key := fieldKey{index, field}
 
 	ret := make([]uint64, len(values))
 
@@ -495,7 +495,7 @@ func (s *TranslateFile) TranslateRowsToUint64(index, frame string, values []stri
 	entry := &LogEntry{
 		Type:  LogEntryTypeInsertRow,
 		Index: []byte(index),
-		Frame: []byte(frame),
+		Field: []byte(field),
 		IDs:   make([]uint64, 0, len(values)),
 		Keys:  make([][]byte, 0, len(values)),
 	}
@@ -524,9 +524,9 @@ func (s *TranslateFile) TranslateRowsToUint64(index, frame string, values []stri
 	return ret, nil
 }
 
-func (s *TranslateFile) TranslateRowToString(index, frame string, id uint64) (string, error) {
+func (s *TranslateFile) TranslateRowToString(index, field string, id uint64) (string, error) {
 	s.mu.RLock()
-	if idx := s.rows[frameKey{index, frame}]; idx != nil {
+	if idx := s.rows[fieldKey{index, field}]; idx != nil {
 		if ret, ok := idx.keyByID(id); ok {
 			s.mu.RUnlock()
 			return string(ret), nil
@@ -548,7 +548,7 @@ func (s *TranslateFile) Reader(ctx context.Context, offset int64) (io.ReadCloser
 type LogEntry struct {
 	Type  uint8
 	Index []byte
-	Frame []byte
+	Field []byte
 
 	IDs  []uint64
 	Keys [][]byte
@@ -558,12 +558,12 @@ type LogEntry struct {
 	Length uint64
 }
 
-// headerSize returns the number of bytes required for size, type, index, frame, & pair count.
+// headerSize returns the number of bytes required for size, type, index, field, & pair count.
 func (e *LogEntry) headerSize() int64 {
 	sz := uVarintSize(e.Length) + // total entry length
 		1 + // type
 		uVarintSize(uint64(len(e.Index))) + len(e.Index) + // Index length and data
-		uVarintSize(uint64(len(e.Frame))) + len(e.Frame) + // Frame length and data
+		uVarintSize(uint64(len(e.Field))) + len(e.Field) + // Field length and data
 		uVarintSize(uint64(len(e.IDs))) // ID/Key pair count
 	return int64(sz)
 }
@@ -604,14 +604,14 @@ func (e *LogEntry) ReadFrom(r io.Reader) (_ int64, err error) {
 		}
 	}
 
-	// Read frame name.
+	// Read field name.
 	if sz, err := binary.ReadUvarint(br); err != nil {
 		return n64, err
 	} else if sz == 0 {
-		e.Frame = nil
+		e.Field = nil
 	} else {
-		e.Frame = make([]byte, sz)
-		if _, err := io.ReadFull(r, e.Frame); err != nil {
+		e.Field = make([]byte, sz)
+		if _, err := io.ReadFull(r, e.Field); err != nil {
 			return n64, err
 		}
 	}
@@ -663,11 +663,11 @@ func (e *LogEntry) WriteTo(w io.Writer) (_ int64, err error) {
 		return 0, err
 	}
 
-	// Write frame name.
-	sz = binary.PutUvarint(b, uint64(len(e.Frame)))
+	// Write field name.
+	sz = binary.PutUvarint(b, uint64(len(e.Field)))
 	if _, err := buf.Write(b[:sz]); err != nil {
 		return 0, err
-	} else if _, err := buf.Write(e.Frame); err != nil {
+	} else if _, err := buf.Write(e.Field); err != nil {
 		return 0, err
 	}
 
@@ -722,9 +722,9 @@ func validLogEntriesLen(p []byte) (n int) {
 	}
 }
 
-type frameKey struct {
+type fieldKey struct {
 	index string
-	frame string
+	field string
 }
 
 const defaultLoadFactor = 90
@@ -910,7 +910,7 @@ type translateFileReader struct {
 	closing chan struct{}
 }
 
-// newTranslateFileReader returns a new instance of TranslateFileReader.
+// newTranslateFileReader returns a new instance of translateFileReader.
 func newTranslateFileReader(ctx context.Context, store *TranslateFile, offset int64) *translateFileReader {
 	return &translateFileReader{
 		ctx:     ctx,

translate_test.go

@@ -4,11 +4,13 @@ import (
 	"bufio"
 	"context"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"math/rand"
 	"os"
 	"reflect"
 	"strconv"
+	"sync"
 	"testing"
 	"time"
 
@@ -134,42 +136,42 @@ func TestTranslateFile_TranslateRow(t *testing.T) {
 	defer s.MustClose()
 
 	// First translation should start id at zero.
-	if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"foo"}); err != nil {
+	if ids, err := s.TranslateRowsToUint64("IDX0", "FIELD0", []string{"foo"}); err != nil {
 		t.Fatal(err)
 	} else if !reflect.DeepEqual(ids, []uint64{1}) {
 		t.Fatalf("unexpected id: %#v", ids)
 	}
 
 	// Next translation on the same index should move to one.
-	if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar"}); err != nil {
+	if ids, err := s.TranslateRowsToUint64("IDX0", "FIELD0", []string{"bar"}); err != nil {
 		t.Fatal(err)
 	} else if !reflect.DeepEqual(ids, []uint64{2}) {
 		t.Fatalf("unexpected id: %#v", ids)
 	}
 
 	// Translation on a different index restarts at 0.
-	if ids, err := s.TranslateRowsToUint64("IDX1", "FRAME0", []string{"bar"}); err != nil {
+	if ids, err := s.TranslateRowsToUint64("IDX1", "FIELD0", []string{"bar"}); err != nil {
 		t.Fatal(err)
 	} else if !reflect.DeepEqual(ids, []uint64{1}) {
 		t.Fatalf("unexpected id: %#v", ids)
 	}
 
-	// Translation on a different frame restarts at 0.
-	if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME1", []string{"bar"}); err != nil {
+	// Translation on a different field restarts at 0.
+	if ids, err := s.TranslateRowsToUint64("IDX0", "FIELD1", []string{"bar"}); err != nil {
 		t.Fatal(err)
 	} else if !reflect.DeepEqual(ids, []uint64{1}) {
 		t.Fatalf("unexpected id: %#v", ids)
 	}
 
 	// Ensure that string values can be looked up by ID.
-	if value, err := s.TranslateRowToString("IDX0", "FRAME0", 2); err != nil {
+	if value, err := s.TranslateRowToString("IDX0", "FIELD0", 2); err != nil {
 		t.Fatal(err)
 	} else if value != "bar" {
 		t.Fatalf("unexpected value: %s", value)
 	}
 
 	// Ensure that non-existent values return blank.
-	if value, err := s.TranslateRowToString("IDX0", "FRAME0", 1000); err != nil {
+	if value, err := s.TranslateRowToString("IDX0", "FIELD0", 1000); err != nil {
 		t.Fatal(err)
 	} else if value != "" {
 		t.Fatalf("unexpected value: %s", value)
@@ -180,22 +182,22 @@ func TestTranslateFile_TranslateRow(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	// Translation on a different frame restarts at 0.
-	if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME1", []string{"bar"}); err != nil {
+	// Translation on a different field restarts at 0.
+	if ids, err := s.TranslateRowsToUint64("IDX0", "FIELD1", []string{"bar"}); err != nil {
 		t.Fatal(err)
 	} else if !reflect.DeepEqual(ids, []uint64{1}) {
 		t.Fatalf("unexpected id: %#v", ids)
 	}
 
 	// Ensure that string values can be looked up by ID.
-	if value, err := s.TranslateRowToString("IDX0", "FRAME0", 2); err != nil {
+	if value, err := s.TranslateRowToString("IDX0", "FIELD0", 2); err != nil {
 		t.Fatal(err)
 	} else if value != "bar" {
 		t.Fatalf("unexpected value: %s", value)
 	}
 
 	// Translate new row and increment sequence.
-	if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"baz"}); err != nil {
+	if ids, err := s.TranslateRowsToUint64("IDX0", "FIELD0", []string{"baz"}); err != nil {
 		t.Fatal(err)
 	} else if !reflect.DeepEqual(ids, []uint64{3}) {
 		t.Fatalf("unexpected id: %#v", ids)
@@ -213,7 +215,7 @@ func TestTranslateFile_TranslateRow_Large(t *testing.T) {
 			keys[j] = strconv.Itoa(i + j + 1)
 		}
 
-		ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", keys)
+		ids, err := s.TranslateRowsToUint64("IDX0", "FIELD0", keys)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -228,7 +230,7 @@ func TestTranslateFile_TranslateRow_Large(t *testing.T) {
 	// Verify values can be returned.
 	for i := 0; i < 1000000; i++ {
 		exp := strconv.Itoa(i + 1)
-		if key, err := s.TranslateRowToString("IDX0", "FRAME0", uint64(i+1)); err != nil {
+		if key, err := s.TranslateRowToString("IDX0", "FIELD0", uint64(i+1)); err != nil {
 			t.Fatal(err)
 		} else if key != exp {
 			t.Fatalf("unexpected key: got=%q, exp=%q", key, exp)
@@ -241,7 +243,7 @@ func TestTranslateFile_TranslateRow_Large(t *testing.T) {
 	}
 	for i := 0; i < 1000000; i++ {
 		exp := strconv.Itoa(i + 1)
-		if key, err := s.TranslateRowToString("IDX0", "FRAME0", uint64(i+1)); err != nil {
+		if key, err := s.TranslateRowToString("IDX0", "FIELD0", uint64(i+1)); err != nil {
 			t.Fatal(err)
 		} else if key != exp {
 			t.Fatalf("unexpected key: got=%q, exp=%q", key, exp)
@@ -255,7 +257,7 @@ func TestTranslateFile_Reader(t *testing.T) {
 		defer s.MustClose()
 		if _, err := s.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil {
 			t.Fatal(err)
-		} else if _, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar", "baz"}); err != nil {
+		} else if _, err := s.TranslateRowsToUint64("IDX0", "FIELD0", []string{"bar", "baz"}); err != nil {
 			t.Fatal(err)
 		}
 
@@ -288,7 +290,7 @@ func TestTranslateFile_Reader(t *testing.T) {
 		} else if diff := cmp.Diff(entry, pilosa.LogEntry{
 			Type:   pilosa.LogEntryTypeInsertRow,
 			Index:  []byte("IDX0"),
-			Frame:  []byte("FRAME0"),
+			Field:  []byte("FIELD0"),
 			IDs:    []uint64{1, 2},
 			Keys:   [][]byte{[]byte("bar"), []byte("baz")},
 			Length: 24,
@@ -327,7 +329,7 @@ func TestTranslateFile_Reader(t *testing.T) {
 		defer s.MustClose()
 		if _, err := s.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil {
 			t.Fatal(err)
-		} else if _, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar", "baz"}); err != nil {
+		} else if _, err := s.TranslateRowsToUint64("IDX0", "FIELD0", []string{"bar", "baz"}); err != nil {
 			t.Fatal(err)
 		}
 
@@ -346,7 +348,7 @@ func TestTranslateFile_Reader(t *testing.T) {
 		} else if diff := cmp.Diff(entry, pilosa.LogEntry{
 			Type:   pilosa.LogEntryTypeInsertRow,
 			Index:  []byte("IDX0"),
-			Frame:  []byte("FRAME0"),
+			Field:  []byte("FIELD0"),
 			IDs:    []uint64{1, 2},
 			Keys:   [][]byte{[]byte("bar"), []byte("baz")},
 			Length: 24,
@@ -356,6 +358,24 @@ func TestTranslateFile_Reader(t *testing.T) {
 	})
 }
 
+func TestPrintTranslateFile(t *testing.T) {
+	// I think this is related to the mmap in s.Open.
+	t.Skip("causes fatal error: fault")
+	f, err := ioutil.TempFile("", "")
+	if err != nil {
+		panic(err)
+	}
+	f.Close()
+
+	s := pilosa.NewTranslateFile()
+	s.Path = f.Name()
+	err = s.Open()
+	if err != nil {
+		t.Fatalf("opening : %v", err)
+	}
+	fmt.Println("blah ", s)
+}
+
 func TestTranslateFile_PrimaryTranslateStore(t *testing.T) {
 	// Create a primary store that accepts writes.
 	primary := MustOpenTranslateFile()
@@ -372,7 +392,7 @@ func TestTranslateFile_PrimaryTranslateStore(t *testing.T) {
 	// Write to the primary.
 	if _, err := primary.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil {
 		t.Fatal(err)
-	} else if _, err := primary.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar", "baz"}); err != nil {
+	} else if _, err := primary.TranslateRowsToUint64("IDX0", "FIELD0", []string{"bar", "baz"}); err != nil {
 		t.Fatal(err)
 	}
 
@@ -385,13 +405,13 @@ func TestTranslateFile_PrimaryTranslateStore(t *testing.T) {
 			return fmt.Errorf("unexpected column 1 value: %s", value)
 		}
 
-		if value, err := replica.TranslateRowToString("IDX0", "FRAME0", 1); err != nil {
+		if value, err := replica.TranslateRowToString("IDX0", "FIELD0", 1); err != nil {
 			return err
 		} else if value != "bar" {
 			return fmt.Errorf("unexpected row 1 value: %s", value)
 		}
 
-		if value, err := replica.TranslateRowToString("IDX0", "FRAME0", 2); err != nil {
+		if value, err := replica.TranslateRowToString("IDX0", "FIELD0", 2); err != nil {
 			return err
 		} else if value != "baz" {
 			return fmt.Errorf("unexpected row 2 value: %s", value)
@@ -495,6 +515,7 @@ func BenchmarkTranslateFile_TranslateColumnToString(b *testing.B) {
 }
 
 type TranslateFile struct {
+	lock sync.Mutex
 	*pilosa.TranslateFile
 }
 
@@ -510,6 +531,12 @@ func NewTranslateFile() *TranslateFile {
 	return s
 }
 
+func (t *TranslateFile) Reader(ctx context.Context, offset int64) (io.ReadCloser, error) {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+	return t.TranslateFile.Reader(ctx, offset)
+}
+
 func MustOpenTranslateFile() *TranslateFile {
 	s := NewTranslateFile()
 	if err := s.Open(); err != nil {
@@ -536,7 +563,9 @@ func (s *TranslateFile) Reopen() error {
 		return err
 	}
 
+	s.lock.Lock()
 	s.TranslateFile = pilosa.NewTranslateFile()
+	s.lock.Unlock()
 	s.Path = prev.Path
 	s.PrimaryTranslateStore = prev.PrimaryTranslateStore
 	if err := s.Open(); err != nil {

utils_internal_test.go

@@ -229,7 +229,7 @@ func (t *ClusterCluster) addCluster(i int, saveTopology bool) (*cluster, error)
 	c.holder = h
 	c.Node = node
 	c.Coordinator = t.common.Nodes[0].ID // the first node is the coordinator
-	c.broadcaster = t
+	c.broadcaster = t.broadcaster(c)
 
 	// add nodes
 	if saveTopology {
@@ -302,39 +302,51 @@ func (t *ClusterCluster) Close() error {
 	return nil
 }
 
-// SendSync is a test implemenetation of Broadcaster SendSync method.
-func (t *ClusterCluster) SendSync(m Message) error {
+type bcast struct {
+	t *ClusterCluster
+	c *cluster
+}
+
+func (b bcast) SendSync(m Message) error {
 	switch obj := m.(type) {
 	case *ClusterStatus:
 		// Apply the send message to all nodes (except the coordinator).
-		for _, c := range t.Clusters {
-			c.mergeClusterStatus(obj)
+		for _, c := range b.t.Clusters {
+			if c != b.c {
+				c.mergeClusterStatus(obj)
+			}
 		}
-		t.mu.RLock()
-		if obj.State == ClusterStateNormal && t.resizing {
-			close(t.resizeDone)
+		b.t.mu.RLock()
+		if obj.State == ClusterStateNormal && b.t.resizing {
+			close(b.t.resizeDone)
 		}
-		t.mu.RUnlock()
+		b.t.mu.RUnlock()
 	}
-
 	return nil
 }
 
+func (t *ClusterCluster) broadcaster(c *cluster) broadcaster {
+	return bcast{
+		t: t,
+		c: c,
+	}
+}
+
 // SendAsync is a test implemenetation of Broadcaster SendAsync method.
-func (t *ClusterCluster) SendAsync(Message) error {
+func (bcast) SendAsync(Message) error {
 	return nil
 }
 
 // SendTo is a test implemenetation of Broadcaster SendTo method.
-func (t *ClusterCluster) SendTo(to *Node, m Message) error {
+func (b bcast) SendTo(to *Node, m Message) error {
 	switch obj := m.(type) {
 	case *ResizeInstruction:
-		err := t.FollowResizeInstruction(obj)
+		err := b.t.FollowResizeInstruction(obj)
 		if err != nil {
 			return err
 		}
 	case *ResizeInstructionComplete:
-		coord := t.clusterByID(to.ID)
+		coord := b.t.clusterByID(to.ID)
 		go coord.markResizeInstructionComplete(obj)
 	}
 	return nil
@@ -404,5 +416,5 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *ResizeInstruction) error
 	}
 
 	node := instr.Coordinator
-	return t.SendTo(node, complete)
+	return bcast{t: t}.SendTo(node, complete)
 }