Merge branch 'master' into internal-85-test-windows-support

Makefile

@@ -139,9 +139,9 @@ gometalinter: require-gometalinter
 	    --enable=ineffassign \
 	    --enable=interfacer \
 	    --enable=maligned \
-	    --enable=megacheck \
 	    --enable=misspell \
 	    --enable=nakedret \
+	    --enable=staticcheck \
 	    --enable=unconvert \
 	    --enable=unparam \
 	    --enable=vet \

cluster.go

@@ -609,8 +609,8 @@ func (c *cluster) addNodeBasicSorted(node *Node) bool {
 // Nodes returns a copy of the slice of nodes in the cluster. Safe for
 // concurrent use, result may be modified.
 func (c *cluster) Nodes() []*Node {
-	c.mu.Lock()
-	defer c.mu.Unlock()
+	c.mu.RLock()
+	defer c.mu.RUnlock()
 	ret := make([]*Node, len(c.nodes))
 	copy(ret, c.nodes)
 	return ret
@@ -1792,7 +1792,7 @@ func (c *cluster) nodeLeave(nodeID string) error {
 	}
 
 	if c.state != ClusterStateNormal && c.state != ClusterStateDegraded {
-		return fmt.Errorf("Cluster must be in state %s to remove a node. Current state: %s",
+		return fmt.Errorf("cluster must be '%s' to remove a node but is '%s'",
 			ClusterStateNormal, c.state)
 	}
 
@@ -1803,7 +1803,7 @@ func (c *cluster) nodeLeave(nodeID string) error {
 
 	// Prevent removing the coordinator node (this node).
 	if nodeID == c.Node.ID {
-		return fmt.Errorf("coordinator cannot be removed; first, make a different node the new coordinator.")
+		return fmt.Errorf("coordinator cannot be removed; first, make a different node the new coordinator")
 	}
 
 	// See if resize job can be generated

cluster_internal_test.go

@@ -83,7 +83,7 @@ func TestFragCombos(t *testing.T) {
 
 // newIndexWithTempPath returns a new instance of Index.
 func newIndexWithTempPath(name string) *Index {
-	path, err := ioutil.TempDir("", "pilosa-index-")
+	path, err := ioutil.TempDir(*TempDir, "pilosa-index-")
 	if err != nil {
 		panic(err)
 	}

ctl/import.go

@@ -151,11 +151,11 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
 func (cmd *ImportCommand) ensureSchema(ctx context.Context) error {
 	err := cmd.client.EnsureIndex(ctx, cmd.Index, cmd.IndexOptions)
 	if err != nil {
-		return fmt.Errorf("Error Creating Index: %s", err)
+		return errors.Wrap(err, "creating index")
 	}
 	err = cmd.client.EnsureFieldWithOptions(ctx, cmd.Index, cmd.Field, cmd.FieldOptions)
 	if err != nil {
-		return fmt.Errorf("Error Creating Field: %s", err)
+		return errors.Wrap(err, "creating field")
 	}
 	return nil
 }

ctl/server.go

@@ -26,6 +26,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
 	flags := cmd.Flags()
 	flags.StringVarP(&srv.Config.DataDir, "data-dir", "d", srv.Config.DataDir, "Directory to store pilosa data files.")
 	flags.StringVarP(&srv.Config.Bind, "bind", "b", srv.Config.Bind, "Default URI on which pilosa should listen.")
+	flags.StringVar(&srv.Config.Advertise, "advertise", srv.Config.Advertise, "Address to advertise externally.")
 	flags.IntVarP(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", "", srv.Config.MaxWritesPerRequest, "Number of write commands per request.")
 	flags.StringVar(&srv.Config.LogPath, "log-path", srv.Config.LogPath, "Log path")
 	flags.BoolVar(&srv.Config.Verbose, "verbose", srv.Config.Verbose, "Enable verbose logging")
@@ -49,6 +50,9 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
 
 	// Gossip
 	flags.StringVarP(&srv.Config.Gossip.Port, "gossip.port", "", srv.Config.Gossip.Port, "Port to which pilosa should bind for internal state sharing.")
+	flags.StringVarP(&srv.Config.Gossip.AdvertiseHost, "gossip.advertise-host", "", srv.Config.Gossip.AdvertiseHost, "Host on which memberlist should advertise.")
+	flags.StringVarP(&srv.Config.Gossip.AdvertisePort, "gossip.advertise-port", "", srv.Config.Gossip.AdvertisePort, "Port on which memberlist should advertise.")
+
 	flags.StringSliceVarP(&srv.Config.Gossip.Seeds, "gossip.seeds", "", srv.Config.Gossip.Seeds, "Host with which to seed the gossip membership.")
 	flags.StringVarP(&srv.Config.Gossip.Key, "gossip.key", "", srv.Config.Gossip.Key, "The path to file of the encryption key for gossip. The contents of the file should be either 16, 24, or 32 bytes to select AES-128, AES-192, or AES-256.")
 	flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.StreamTimeout), "gossip.stream-timeout", "", (time.Duration)(srv.Config.Gossip.StreamTimeout), "Timeout for establishing a stream connection with a remote node for a full state sync.")

diagnostics.go

@@ -137,11 +137,11 @@ func (d *diagnosticsCollector) compareVersion(value string) error {
 	localVersion := versionSegments(d.version)
 
 	if localVersion[0] < currentVersion[0] { //Major
-		return fmt.Errorf("Warning: You are running Pilosa %s. A newer version (%s) is available: https://github.com/pilosa/pilosa/releases", d.version, value)
+		return fmt.Errorf("you are running Pilosa %s, a newer version (%s) is available: https://github.com/pilosa/pilosa/releases", d.version, value)
 	} else if localVersion[1] < currentVersion[1] && localVersion[0] == currentVersion[0] { // Minor
-		return fmt.Errorf("Warning: You are running Pilosa %s. The latest Minor release is %s: https://github.com/pilosa/pilosa/releases", d.version, value)
+		return fmt.Errorf("you are running Pilosa %s, the latest minor release is %s: https://github.com/pilosa/pilosa/releases", d.version, value)
 	} else if localVersion[2] < currentVersion[2] && localVersion[0] == currentVersion[0] && localVersion[1] == currentVersion[1] { // Patch
-		return fmt.Errorf("There is a new patch release of Pilosa available: %s: https://github.com/pilosa/pilosa/releases", value)
+		return fmt.Errorf("there is a new patch release of Pilosa available: %s: https://github.com/pilosa/pilosa/releases", value)
 	}
 
 	return nil

diagnostics_internal_test.go

@@ -82,19 +82,19 @@ func TestDiagnosticsVersion_Compare(t *testing.T) {
 	d.SetVersion(version)
 
 	err := d.compareVersion("v1.7.0")
-	if !strings.Contains(err.Error(), "A newer version") {
+	if !strings.Contains(err.Error(), "a newer version") {
 		t.Fatalf("Expected a newer version is available, actual error: %s", err)
 	}
 	err = d.compareVersion("1.7.0")
-	if !strings.Contains(err.Error(), "A newer version") {
+	if !strings.Contains(err.Error(), "a newer version") {
 		t.Fatalf("Expected a newer version is available, actual error: %s", err)
 	}
 	err = d.compareVersion("0.7.0")
-	if !strings.Contains(err.Error(), "The latest Minor release is") {
+	if !strings.Contains(err.Error(), "the latest minor release is") {
 		t.Fatalf("Expected Minor Version Missmatch, actual error: %s", err)
 	}
 	err = d.compareVersion("0.1.2")
-	if !strings.Contains(err.Error(), "There is a new patch release of Pilosa") {
+	if !strings.Contains(err.Error(), "there is a new patch release of Pilosa") {
 		t.Fatalf("Expected Patch Version Missmatch, actual error: %s", err)
 	}
 	err = d.compareVersion("0.1.1")

docs/configuration.md

@@ -36,6 +36,17 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h
 
 ### All Options
 
+#### Advertise
+
+* Description: Address advertised by the server to other nodes in the cluster and to clients via the `/status` endpoint. Host defaults to the IP address represented by `bind` and port to 10101. If `bind` is set to `0.0.0.0` and `advertise` is not specified, then Pilosa will try to determine a reasonable, external IP address to use for `advertise`.
+* Flag: `--advertise="192.168.1.100:10101"`
+* Env: `PILOSA_BIND="192.168.1.100:10101"`
+* Config:
+
+    ```toml
+    advertise = 192.168.1.100:10101
+    ```
+
 #### Anti Entropy Interval
 
 * Description: Interval at which the cluster will run its anti-entropy routine which ensures that all replicas of each fragment are in sync.
@@ -50,7 +61,7 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h
 
 #### Bind
 
-* Description: host:port on which the Pilosa server will listen for requests. Host defaults to localhost and port to 10101.
+* Description: host:port on which the Pilosa server will listen for requests. Host defaults to localhost and port to 10101. If `bind` is set to `0.0.0.0` then Pilosa will listen on all available interfaces.
 * Flag: `--bind="localhost:10101"`
 * Env: `PILOSA_BIND="localhost:10101"`
 * Config:
@@ -115,6 +126,30 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h
     max-writes-per-request = 5000
     ```
 
+#### Gossip Advertise Host
+
+* Description: Host on which memberlist should advertise. Defaults to `advertise` host.
+* Flag: `--gossip.advertise-host=192.168.1.100`
+* Env: `PILOSA_GOSSIP_ADVERTISE_HOST=192.168.1.100
+* Config:
+
+    ```toml
+    [gossip]
+      advertise-host = 192.168.1.100
+    ```
+
+#### Gossip Advertise Port
+
+* Description: Port on which memberlist should advertise. Defaults to `advertise` port.
+* Flag: `--gossip.advertise-port=15001`
+* Env: `PILOSA_GOSSIP_ADVERTISE_PORT=15001`
+* Config:
+
+    ```toml
+    [gossip]
+      advertise-port = 15001
+    ```
+
 #### Gossip Port
 
 * Description: Port to which Pilosa should bind for internal communication. If more than one Pilosa server is running on the same host, the gossip port for each server must be unique.
@@ -366,7 +401,7 @@ A three node cluster running on different hosts could be minimally configured as
     
     [gossip]
       port = 12000
-      seed = "node0.pilosa.com:12000"
+      seeds = ["node0.pilosa.com:12000"]
 
     [cluster]
       replicas = 1
@@ -379,7 +414,7 @@ A three node cluster running on different hosts could be minimally configured as
     
     [gossip]
       port = 12000
-      seed = "node0.pilosa.com:12000"
+      seeds = ["node0.pilosa.com:12000"]
 
     [cluster]
       replicas = 1
@@ -392,7 +427,7 @@ A three node cluster running on different hosts could be minimally configured as
     
     [gossip]
       port = 12000
-      seed = "node0.pilosa.com:12000"
+      seeds = ["node0.pilosa.com:12000"]
 
     [cluster]
       replicas = 1
@@ -410,7 +445,7 @@ The same cluster which uses HTTPS instead of HTTP can be configured as follows.
 
     [gossip]
       port = 12000
-      seed = "node0.pilosa.com:12000"
+      seeds = ["node0.pilosa.com:12000"]
       key = "/home/pilosa/private/gossip.key32"
 
     [cluster]
@@ -428,7 +463,7 @@ The same cluster which uses HTTPS instead of HTTP can be configured as follows.
 
     [gossip]
       port = 12000
-      seed = "node0.pilosa.com:12000"
+      seeds = ["node0.pilosa.com:12000"]
       key = "/home/pilosa/private/gossip.key32"
 
     [cluster]
@@ -446,7 +481,7 @@ The same cluster which uses HTTPS instead of HTTP can be configured as follows.
 
     [gossip]
       port = 12000
-      seed = "node0.pilosa.com:12000"
+      seeds = ["node0.pilosa.com:12000"]
       key = "/home/pilosa/private/gossip.key32"
 
     [cluster]
@@ -468,7 +503,7 @@ You can run a cluster on the same host using the configuration above with a few
 
     [gossip]
       port = 12000
-      seed = "localhost:12000"
+      seeds = ["localhost:12000"]
       key = "/home/pilosa/private/gossip.key32"
 
     [cluster]
@@ -486,7 +521,7 @@ You can run a cluster on the same host using the configuration above with a few
 
     [gossip]
       port = 12001
-      seed = "localhost:12000"
+      seeds = ["localhost:12000"]
       key = "/home/pilosa/private/gossip.key32"
 
     [cluster]
@@ -504,7 +539,7 @@ You can run a cluster on the same host using the configuration above with a few
 
     [gossip]
       port = 12002
-      seed = "localhost:12000"
+      seeds = ["localhost:12000"]
       key = "/home/pilosa/private/gossip.key32"
 
     [cluster]

docs/query-language.md

@@ -594,6 +594,34 @@ Count(Row(stargazer=1))
 
 * Result is the number of repositories that user 1 has starred.
 
+#### Shift
+**Spec:**
+
+```
+Shift(<ROW_CALL>, [n=UINT])
+```
+
+**Description:**
+
+Returns the row specified by `ROW_CALL` shifted by `n` bits.
+
+**Result Type:** object with attrs and columns
+
+attrs will always be empty
+
+**Examples:**
+
+Query all columns with a bit set in row 1 of the field `stargazer`
+and shift the result by 2:
+```request
+Shift(Row(stargazer=1), n=2)
+```
+```response
+{"attrs":{},"columns":[12, 22]}
+```
+
+* columns are the repositories which user 1 has starred shifted by 2 bits.
+
 #### TopN
 
 **Spec:**
@@ -786,7 +814,7 @@ Options(Row(f1=10), shards=[0, 2])
 **Spec:**
 
 ```
-Rows(field=<STRING>, previous=<UINT|STRING>, limit=<UINT>, column=<UINT|STRING>)
+Rows(<FIELD>, previous=<UINT|STRING>, limit=<UINT>, column=<UINT|STRING>)
 ```
 
 **Description:**
@@ -810,7 +838,7 @@ result of the previous request will start from the next available row.
 
 Without keys:
 ```request
-Rows(field=blah)
+Rows(blah)
 ```
 ```response
 {"rows":[1,9,39]}
@@ -818,7 +846,7 @@ Rows(field=blah)
 
 With keys:
 ```request
-Rows(field=blahk)
+Rows(blahk)
 ```
 ```response
 {"rows":null,"keys":["haha","zaaa","traa"]}
@@ -859,7 +887,7 @@ specify the field and row for each row that was intersected to get that result.
 
 A single `Rows` query.
 ```request
-GroupBy(Rows(field=blah))
+GroupBy(Rows(blah))
 ```
 ```response
 [{"group":[{"field":"blah","rowID":1}],"count":1},
@@ -869,7 +897,7 @@ GroupBy(Rows(field=blah))
 
 With two `Rows` queries - one with IDs and one with keys.
 ```request
-GroupBy(Rows(field=blah), Rows(field=blahk), limit=7)
+GroupBy(Rows(blah), Rows(blahk), limit=7)
 ```
 ```response
 [{"group":[{"field":"blah","rowID":1},{"field":"blahk","rowKey":"haha"}],"count":1},
@@ -883,7 +911,7 @@ GroupBy(Rows(field=blah), Rows(field=blahk), limit=7)
 
 Getting the rest of the results from the previous example (paging).
 ```request
-GroupBy(Rows(field=blah, previous=39), Rows(field=blahk, previous="haha"), limit=7)
+GroupBy(Rows(blah, previous=39), Rows(blahk, previous="haha"), limit=7)
 ```
 
 ```response

executor.go

@@ -18,7 +18,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"log"
 	"sort"
 	"time"
 
@@ -559,6 +558,8 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, index string, c *
 		return e.executeXorShard(ctx, index, c, shard)
 	case "Not":
 		return e.executeNotShard(ctx, index, c, shard)
+	case "Shift":
+		return e.executeShiftShard(ctx, index, c, shard)
 	default:
 		return nil, fmt.Errorf("unknown call: %s", c.Name)
 	}
@@ -806,7 +807,7 @@ func (e *executor) executeTopNShard(ctx context.Context, index string, c *pql.Ca
 		return nil, nil
 	}
 
-	if minThreshold <= 0 {
+	if minThreshold == 0 {
 		minThreshold = defaultMinThreshold
 	}
 
@@ -915,6 +916,12 @@ func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call
 	// TODO support TopN in here would be really cool - and pretty easy I think.
 	childRows := make([]RowIDs, len(c.Children))
 	for i, child := range c.Children {
+		// Check "field" first for backwards compatibility, then set _field.
+		// TODO: remove at Pilosa 2.0
+		if fieldName, ok := child.Args["field"].(string); ok {
+			child.Args["_field"] = fieldName
+		}
+
 		if child.Name != "Rows" {
 			return nil, errors.Errorf("'%s' is not a valid child query for GroupBy, must be 'Rows'", child.Name)
 		}
@@ -1089,6 +1096,17 @@ func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql
 }
 
 func (e *executor) executeRows(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (RowIDs, error) {
+	// Fetch field name from argument.
+	// Check "field" first for backwards compatibility.
+	// TODO: remove at Pilosa 2.0
+	var fieldName string
+	var ok bool
+	if fieldName, ok = c.Args["field"].(string); ok {
+		c.Args["_field"] = fieldName
+	}
+	if fieldName, ok = c.Args["_field"].(string); !ok {
+		return nil, errors.New("Rows() field required")
+	}
 	if columnID, ok, err := c.UintArg("column"); err != nil {
 		return nil, errors.Wrap(err, "getting column")
 	} else if ok {
@@ -1097,7 +1115,7 @@ func (e *executor) executeRows(ctx context.Context, index string, c *pql.Call, s
 
 	// Execute calls in bulk on each remote node and merge.
 	mapFn := func(shard uint64) (interface{}, error) {
-		return e.executeRowsShard(ctx, index, c, shard)
+		return e.executeRowsShard(ctx, index, fieldName, c, shard)
 	}
 
 	// Determine limit so we can use it when reducing.
@@ -1122,22 +1140,25 @@ func (e *executor) executeRows(ctx context.Context, index string, c *pql.Call, s
 	return results, nil
 }
 
-func (e *executor) executeRowsShard(_ context.Context, index string, c *pql.Call, shard uint64) (RowIDs, error) {
+func (e *executor) executeRowsShard(_ context.Context, index string, fieldName string, c *pql.Call, shard uint64) (RowIDs, error) {
 	// Fetch index.
 	idx := e.Holder.Index(index)
 	if idx == nil {
 		return nil, ErrIndexNotFound
 	}
-	// Fetch field name from argument.
-	fieldName, ok := c.Args["field"].(string)
-	if !ok {
-		return nil, errors.New("Rows() argument required: field")
-	}
 	// Fetch field.
 	f := e.Holder.Field(index, fieldName)
 	if f == nil {
 		return nil, ErrFieldNotFound
 	}
+
+	// Rows query does not currently support a `time` field that has
+	// `noStandardView: true`.
+	// TODO https://github.com/pilosa/pilosa/issues/1783
+	if f.Type() == FieldTypeTime && f.options.NoStandardView {
+		return nil, errors.New("Rows() query on time field with no standard view is not currently supported")
+	}
+
 	frag := e.Holder.fragment(index, fieldName, viewStandard, shard)
 	if frag == nil {
 		return make(RowIDs, 0), nil
@@ -1174,7 +1195,7 @@ func (e *executor) executeRowShard(ctx context.Context, index string, c *pql.Cal
 	defer span.Finish()
 
 	if c.Name == "Range" {
-		log.Print("DEPRECATED: Range() is deprecated, please use Row() instead.")
+		e.Holder.Logger.Printf("DEPRECATED: Range() is deprecated, please use Row() instead.")
 	}
 
 	// Handle bsiGroup ranges differently.
@@ -1508,6 +1529,27 @@ func (e *executor) executeNotShard(ctx context.Context, index string, c *pql.Cal
 	return existenceRow.Difference(row), nil
 }
 
+// executeShiftShard executes a shift() call for a local shard.
+func (e *executor) executeShiftShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
+	n, _, err := c.IntArg("n")
+	if err != nil {
+		return nil, fmt.Errorf("executeShiftShard: %v", err)
+	}
+
+	if len(c.Children) == 0 {
+		return nil, errors.New("Shift() requires an input row")
+	} else if len(c.Children) > 1 {
+		return nil, errors.New("Shift() only accepts a single row input")
+	}
+
+	row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard)
+	if err != nil {
+		return nil, err
+	}
+
+	return row.Shift(n)
+}
+
 // executeCount executes a count() call.
 func (e *executor) executeCount(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (uint64, error) {
 	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeCount")
@@ -1568,14 +1610,14 @@ func (e *executor) executeClearBit(ctx context.Context, index string, c *pql.Cal
 	if err != nil {
 		return false, fmt.Errorf("reading Clear() row: %v", err)
 	} else if !ok {
-		return false, fmt.Errorf("Clear() row argument '%v' required", rowLabel)
+		return false, fmt.Errorf("row=<row> argument required to Clear() call")
 	}
 
 	colID, ok, err := c.UintArg("_" + columnLabel)
 	if err != nil {
 		return false, fmt.Errorf("reading Clear() column: %v", err)
 	} else if !ok {
-		return false, fmt.Errorf("Clear() col argument '%v' required", columnLabel)
+		return false, fmt.Errorf("column argument to Clear(<COLUMN>, <FIELD>=<ROW>) required")
 	}
 
 	return e.executeClearBitField(ctx, index, c, f, colID, rowID, opt)
@@ -1694,19 +1736,19 @@ func (e *executor) executeClearRowShard(ctx context.Context, index string, c *pq
 	return changed, nil
 }
 
-// executeSetRow executes a SetRow() call.
+// executeSetRow executes a Store() call.
 func (e *executor) executeSetRow(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (bool, error) {
-	// Ensure the field type supports SetRow().
+	// Ensure the field type supports Store().
 	fieldName, err := c.FieldArg()
 	if err != nil {
-		return false, errors.New("SetRow() argument required: field")
+		return false, errors.New("field required for Store()")
 	}
 	field := e.Holder.Field(index, fieldName)
 	if field == nil {
 		return false, ErrFieldNotFound
 	}
 	if field.Type() != FieldTypeSet {
-		return false, fmt.Errorf("SetRow() is not supported on %s field types", field.Type())
+		return false, fmt.Errorf("can't Store() on a %s field", field.Type())
 	}
 
 	// Execute calls in bulk on each remote node and merge.
@@ -1731,15 +1773,15 @@ func (e *executor) executeSetRow(ctx context.Context, index string, c *pql.Call,
 func (e *executor) executeSetRowShard(ctx context.Context, index string, c *pql.Call, shard uint64) (bool, error) {
 	fieldName, err := c.FieldArg()
 	if err != nil {
-		return false, errors.New("SetRow() argument required: field")
+		return false, errors.New("Store() argument required: field")
 	}
 
 	// Read fields using labels.
 	rowID, ok, err := c.UintArg(fieldName)
 	if err != nil {
-		return false, fmt.Errorf("reading SetRow() row: %v", err)
+		return false, fmt.Errorf("reading Store() row: %v", err)
 	} else if !ok {
-		return false, fmt.Errorf("SetRow() row argument '%v' required", rowLabel)
+		return false, fmt.Errorf("need the <FIELD>=<ROW> argument on Store()")
 	}
 
 	field := e.Holder.Field(index, fieldName)
@@ -1756,7 +1798,7 @@ func (e *executor) executeSetRowShard(ctx context.Context, index string, c *pql.
 		}
 		src = row
 	} else {
-		return false, errors.New("SetRow() requires a source row")
+		return false, errors.New("Store() requires a source row")
 	}
 
 	// Set the row on the standard view.
@@ -1775,7 +1817,7 @@ func (e *executor) executeSetRowShard(ctx context.Context, index string, c *pql.
 	}
 	set, err := fragment.setRow(src, rowID)
 	if err != nil {
-		return false, errors.Wrapf(err, "setting row %d on view %s shard %d", rowID, viewStandard, shard)
+		return false, errors.Wrapf(err, "storing row %d on view %s shard %d", rowID, viewStandard, shard)
 	}
 	changed = changed || set
 
@@ -2336,7 +2378,7 @@ func (e *executor) translateCall(index string, idx *Index, c *pql.Call) error {
 		rowKey = "_" + rowLabel
 		fieldName = callArgString(c, "_field")
 	case "Rows":
-		fieldName = callArgString(c, "field")
+		fieldName = callArgString(c, "_field")
 		rowKey = "previous"
 		colKey = "column"
 	case "GroupBy":
@@ -2441,7 +2483,7 @@ func (e *executor) translateGroupByCall(index string, idx *Index, c *pql.Call) e
 
 	fields := make([]*Field, len(c.Children))
 	for i, child := range c.Children {
-		fieldname := callArgString(child, "field")
+		fieldname := callArgString(child, "_field")
 		field := idx.Field(fieldname)
 		if field == nil {
 			return errors.Wrapf(ErrFieldNotFound, "getting field '%s' from '%s'", fieldname, child)
@@ -2552,7 +2594,7 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res
 	case RowIDs:
 		other := RowIdentifiers{}
 
-		fieldName := callArgString(call, "field")
+		fieldName := callArgString(call, "_field")
 		if fieldName == "" {
 			return nil, ErrFieldNotFound
 		}
@@ -2750,11 +2792,12 @@ func newGroupByIterator(rowIDs []RowIDs, children []*pql.Call, filter *Row, inde
 		fields: make([]FieldRow, len(children)),
 	}
 
+	var fieldName string
+	var ok bool
 	ignorePrev := false
 	for i, call := range children {
-		fieldName, ok := call.Args["field"].(string)
-		if !ok {
-			return nil, errors.Errorf("%s call must have 'field' argument with valid (string) field name. Got %v of type %[2]T", call.Name, call.Args["field"])
+		if fieldName, ok = call.Args["_field"].(string); !ok {
+			return nil, errors.Errorf("%s call must have field with valid (string) field name. Got %v of type %[2]T", call.Name, call.Args["_field"])
 		}
 		if holder.Field(index, fieldName) == nil {
 			return nil, ErrFieldNotFound

executor_internal_test.go

@@ -14,7 +14,7 @@ func TestExecutor_TranslateGroupByCall(t *testing.T) {
 	e := &executor{
 		Holder: NewHolder(),
 	}
-	e.Holder.Path, _ = ioutil.TempDir("", "")
+	e.Holder.Path, _ = ioutil.TempDir(*TempDir, "")
 	err := e.Holder.Open()
 	if err != nil {
 		t.Fatalf("opening holder: %v", err)
@@ -46,7 +46,7 @@ func TestExecutor_TranslateGroupByCall(t *testing.T) {
 		t.Fatalf("translating rows %v, %v", erra, errb)
 	}
 
-	query, err := pql.ParseString(`GroupBy(Rows(field=ak), Rows(field=b), Rows(field=ck), previous=["la", 0, "ha"])`)
+	query, err := pql.ParseString(`GroupBy(Rows(ak), Rows(b), Rows(ck), previous=["la", 0, "ha"])`)
 	if err != nil {
 		t.Fatalf("parsing query: %v", err)
 	}
@@ -69,28 +69,28 @@ func TestExecutor_TranslateGroupByCall(t *testing.T) {
 		err string
 	}{
 		{
-			pql: `GroupBy(Rows(field=notfound), previous=1)`,
+			pql: `GroupBy(Rows(notfound), previous=1)`,
 			err: "'previous' argument must be list",
 		},
 		{
-			pql: `GroupBy(Rows(field=ak), previous=["la", 0])`,
+			pql: `GroupBy(Rows(ak), previous=["la", 0])`,
 			err: "mismatched lengths",
 		},
 		{
-			pql: `GroupBy(Rows(field=ak), previous=[1])`,
+			pql: `GroupBy(Rows(ak), previous=[1])`,
 			err: "prev value must be a string",
 		},
 		{
-			pql: `GroupBy(Rows(field=notfound), previous=[1])`,
+			pql: `GroupBy(Rows(notfound), previous=[1])`,
 			err: ErrFieldNotFound.Error(),
 		},
 		// TODO: an unknown key will actually allocate an id. this is probably bad.
 		// {
-		// 	pql: `GroupBy(Rows(field=ak), previous=["zoop"])`,
+		// 	pql: `GroupBy(Rows(ak), previous=["zoop"])`,
 		// 	err: "translating row key '",
 		// },
 		{
-			pql: `GroupBy(Rows(field=b), previous=["la"])`,
+			pql: `GroupBy(Rows(b), previous=["la"])`,
 			err: "which doesn't use string keys",
 		},
 	}

executor_test.go

@@ -16,7 +16,9 @@ package pilosa_test
 
 import (
 	"context"
+	"flag"
 	"fmt"
+	"io/ioutil"
 	"math/rand"
 	"reflect"
 	"strconv"
@@ -33,6 +35,22 @@ import (
 	"github.com/pkg/errors"
 )
 
+var (
+	TempDir = getTempDirString()
+)
+
+func getTempDirString() (td *string) {
+	tdflag := flag.Lookup("temp-dir")
+
+	if tdflag == nil {
+		td = flag.String("temp-dir", "", "Directory in which to place temporary data (e.g. for benchmarking). Useful if you are trying to benchmark different storage configurations.")
+	} else {
+		s := tdflag.Value.String()
+		td = &s
+	}
+	return td
+}
+
 // Ensure a row query can be executed.
 func TestExecutor_Execute_Row(t *testing.T) {
 	t.Run("RowIDColumnID", func(t *testing.T) {
@@ -2283,7 +2301,7 @@ Set(4500001, fn=4)
 	t.Run("remote groupBy", func(t *testing.T) {
 		if res, err := c[1].API.Query(context.Background(), &pilosa.QueryRequest{
 			Index: "i",
-			Query: `GroupBy(Rows(field=f))`,
+			Query: `GroupBy(Rows(f))`,
 		}); err != nil {
 			t.Fatalf("GroupBy querying: %v", err)
 		} else {
@@ -2987,7 +3005,16 @@ func TestExecutor_Execute_SetRow(t *testing.T) {
 }
 
 func benchmarkExistence(nn bool, b *testing.B) {
-	c := test.MustRunCluster(b, 1)
+	c := test.MustNewCluster(b, 1)
+	var err error
+	c[0].Config.DataDir, err = ioutil.TempDir(*TempDir, "benchmarkExistence")
+	if err != nil {
+		b.Fatalf("getting temp dir: %v", err)
+	}
+	err = c.Start()
+	if err != nil {
+		b.Fatalf("starting cluster: %v", err)
+	}
 	defer c.Close()
 	hldr := test.Holder{Holder: c[0].Server.Holder()}
 
@@ -3038,27 +3065,45 @@ func TestExecutor_Execute_Rows(t *testing.T) {
 		{13, 3},
 	})
 
-	rows := c.Query(t, "i", `Rows(field=general)`).Results[0].(pilosa.RowIdentifiers)
+	rows := c.Query(t, "i", `Rows(general)`).Results[0].(pilosa.RowIdentifiers)
 	if !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{10, 11, 12, 13}}) {
 		t.Fatalf("unexpected rows: %+v", rows)
 	}
 
-	rows = c.Query(t, "i", `Rows(field=general, limit=2)`).Results[0].(pilosa.RowIdentifiers)
+	// backwards compatibility
+	// TODO: remove at Pilosa 2.0
+	rows = c.Query(t, "i", `Rows(field=general)`).Results[0].(pilosa.RowIdentifiers)
+	if !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{10, 11, 12, 13}}) {
+		t.Fatalf("unexpected rows: %+v", rows)
+	}
+
+	rows = c.Query(t, "i", `Rows(general, limit=2)`).Results[0].(pilosa.RowIdentifiers)
 	if !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{10, 11}}) {
 		t.Fatalf("unexpected rows: %+v", rows)
 	}
 
-	rows = c.Query(t, "i", `Rows(field=general, previous=10,limit=2)`).Results[0].(pilosa.RowIdentifiers)
+	rows = c.Query(t, "i", `Rows(general, previous=10,limit=2)`).Results[0].(pilosa.RowIdentifiers)
 	if !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{11, 12}}) {
 		t.Fatalf("unexpected rows: %+v", rows)
 	}
 
-	rows = c.Query(t, "i", `Rows(field=general, column=2)`).Results[0].(pilosa.RowIdentifiers)
+	rows = c.Query(t, "i", `Rows(general, column=2)`).Results[0].(pilosa.RowIdentifiers)
 	if !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{11, 12}}) {
 		t.Fatalf("unexpected rows: %+v", rows)
 	}
 }
 
+func TestExecutor_Execute_RowsTime(t *testing.T) {
+	c := test.MustRunCluster(t, 1)
+	defer c.Close()
+	c.CreateField(t, "i", pilosa.IndexOptions{}, "t", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), true))
+
+	exp := "executing: Rows() query on time field with no standard view is not currently supported"
+	if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Rows(field=t)`}); err == nil || err.Error() != exp {
+		t.Fatalf("expected error: %s", exp)
+	}
+}
+
 func TestExecutor_Execute_Query_Error(t *testing.T) {
 	c := test.MustRunCluster(t, 1)
 	defer c.Close()
@@ -3070,35 +3115,27 @@ func TestExecutor_Execute_Query_Error(t *testing.T) {
 	}{
 		{
 			query: "GroupBy(Rows())",
-			error: "Rows call must have 'field' argument",
+			error: "Rows call must have field",
 		},
 		{
-			query: "GroupBy(Rows(field=true))",
-			error: "Rows call must have 'field' argument",
+			query: "GroupBy(Rows(\"true\"))",
+			error: "parsing: parsing:",
 		},
 		{
-			query: "GroupBy(Rows(field=\"true\"))",
-			error: "field not found",
+			query: "GroupBy(Rows(1))",
+			error: "parsing: parsing:",
 		},
 		{
-			query: "GroupBy(Rows(field=1))",
-			error: "Rows call must have 'field' argument",
-		},
-		{
-			query: "GroupBy(Rows(field))",
-			error: "parse error",
-		},
-		{
-			query: "GroupBy(Rows(field=general, limit=-1))",
+			query: "GroupBy(Rows(general, limit=-1))",
 			error: "must be positive, but got",
 		},
 		{
-			query: "GroupBy(Rows(field=general), limit=-1)",
+			query: "GroupBy(Rows(general), limit=-1)",
 			error: "must be positive, but got",
 		},
 		{
-			query: "GroupBy(Rows(field=general), filter=Rows(field=general))",
-			error: "unknown call: Rows",
+			query: "GroupBy(Rows(general), filter=Rows(general))",
+			error: "parsing: parsing:",
 		},
 	}
 
@@ -3112,7 +3149,7 @@ func TestExecutor_Execute_Query_Error(t *testing.T) {
 				t.Fatalf("should have gotten an error on invalid rows query, but got %#v", r)
 			}
 			if !strings.Contains(err.Error(), test.error) {
-				t.Fatalf("unexpected error message: %s", err.Error())
+				t.Fatalf("unexpected error message:\n%s != %s", test.error, err.Error())
 			}
 		})
 	}
@@ -3158,68 +3195,80 @@ func TestExecutor_Execute_Rows_Keys(t *testing.T) {
 		q   string
 		exp []string
 	}{
+		{
+			q:   `Rows(f)`,
+			exp: []string{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18"},
+		},
+		// backwards compatibility
+		// TODO: remove at Pilosa 2.0
 		{
 			q:   `Rows(field=f)`,
 			exp: []string{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18"},
 		},
+		{
+			q:   `Rows(f, limit=2)`,
+			exp: []string{"0", "1"},
+		},
+		// backwards compatibility
+		// TODO: remove at Pilosa 2.0
 		{
 			q:   `Rows(field=f, limit=2)`,
 			exp: []string{"0", "1"},
 		},
 		{
-			q:   `Rows(field=f, previous="15")`,
+			q:   `Rows(f, previous="15")`,
 			exp: []string{"16", "17", "18"},
 		},
 		{
-			q:   `Rows(field=f, previous="11", limit=2)`,
+			q:   `Rows(f, previous="11", limit=2)`,
 			exp: []string{"12", "13"},
 		},
 		{
-			q:   `Rows(field=f, previous="17", limit=5)`,
+			q:   `Rows(f, previous="17", limit=5)`,
 			exp: []string{"18"},
 		},
 		{
-			q:   `Rows(field=f, previous="18")`,
+			q:   `Rows(f, previous="18")`,
 			exp: []string{},
 		},
 		{
-			q:   `Rows(field=f, previous="1", limit=0)`,
+			q:   `Rows(f, previous="1", limit=0)`,
 			exp: []string{},
 		},
 		{
-			q:   `Rows(field=f, column="1")`,
+			q:   `Rows(f, column="1")`,
 			exp: []string{"0", "1"},
 		},
 		{
-			q:   `Rows(field=f, column="2")`,
+			q:   `Rows(f, column="2")`,
 			exp: []string{"0", "1", "2"},
 		},
 		{
-			q:   `Rows(field=f, column="3")`,
+			q:   `Rows(f, column="3")`,
 			exp: []string{"1", "2", "3"},
 		},
 		{
-			q:   `Rows(field=f, limit=2, column="3")`,
+			q:   `Rows(f, limit=2, column="3")`,
 			exp: []string{"1", "2"},
 		},
 		{
-			q:   fmt.Sprintf(`Rows(field=f, previous="15", column="%d")`, ShardWidth*9+17),
+			q:   fmt.Sprintf(`Rows(f, previous="15", column="%d")`, ShardWidth*9+17),
 			exp: []string{"16", "17"},
 		},
 		{
-			q:   fmt.Sprintf(`Rows(field=f, previous="11", limit=2, column="%d")`, ShardWidth*5+14),
+			q:   fmt.Sprintf(`Rows(f, previous="11", limit=2, column="%d")`, ShardWidth*5+14),
 			exp: []string{"12", "13"},
 		},
 		{
-			q:   fmt.Sprintf(`Rows(field=f, previous="17", limit=5, column="%d")`, ShardWidth*9+18),
+			q:   fmt.Sprintf(`Rows(f, previous="17", limit=5, column="%d")`, ShardWidth*9+18),
 			exp: []string{"18"},
 		},
 		{
-			q:   `Rows(field=f, previous="18", column="19")`,
+			q:   `Rows(f, previous="18", column="19")`,
 			exp: []string{},
 		},
 		{
-			q:   `Rows(field=f, previous="1", limit=0, column="0")`,
+			q:   `Rows(f, previous="1", limit=0, column="0")`,
 			exp: []string{},
 		},
 	}
@@ -3271,13 +3320,27 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
 		})
 
 		t.Run("Unknown Field ", func(t *testing.T) {
-			if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(Rows(field=missing))`}); err != nil {
+			if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(Rows(missing))`}); err != nil {
 				if errors.Cause(err) != pilosa.ErrFieldNotFound {
 					t.Fatalf("unexpected error\n\"%s\" not returned instead \n\"%s\"", pilosa.ErrFieldNotFound, err)
 				}
 			}
 		})
 
+		// backwards compatibility
+		// TODO: remove at Pilosa 2.0
+		t.Run("BasicLegacy", func(t *testing.T) {
+			expected := []pilosa.GroupCount{
+				{Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 100}}, Count: 3},
+				{Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 110}}, Count: 1},
+				{Group: []pilosa.FieldRow{{Field: "general", RowID: 11}, {Field: "sub", RowID: 110}}, Count: 1},
+				{Group: []pilosa.FieldRow{{Field: "general", RowID: 12}, {Field: "sub", RowID: 110}}, Count: 1},
+			}
+
+			results := c.Query(t, "i", `GroupBy(Rows(field=general), Rows(sub))`).Results[0].([]pilosa.GroupCount)
+			test.CheckGroupBy(t, expected, results)
+		})
+
 		t.Run("Basic", func(t *testing.T) {
 			expected := []pilosa.GroupCount{
 				{Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 100}}, Count: 3},
@@ -3286,7 +3349,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
 				{Group: []pilosa.FieldRow{{Field: "general", RowID: 12}, {Field: "sub", RowID: 110}}, Count: 1},
 			}
 
-			results := c.Query(t, "i", `GroupBy(Rows(field=general), Rows(field=sub))`).Results[0].([]pilosa.GroupCount)
+			results := c.Query(t, "i", `GroupBy(Rows(general), Rows(sub))`).Results[0].([]pilosa.GroupCount)
 			test.CheckGroupBy(t, expected, results)
 		})
 
@@ -3296,7 +3359,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
 				{Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 110}}, Count: 1},
 			}
 
-			results := c.Query(t, "i", `GroupBy(Rows(field=general), Rows(field=sub), filter=Row(general=10))`).Results[0].([]pilosa.GroupCount)
+			results := c.Query(t, "i", `GroupBy(Rows(general), Rows(sub), filter=Row(general=10))`).Results[0].([]pilosa.GroupCount)
 			test.CheckGroupBy(t, expected, results)
 		})
 
@@ -3306,7 +3369,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
 				{Group: []pilosa.FieldRow{{Field: "general", RowID: 12}}, Count: 2},
 			}
 
-			results := c.Query(t, "i", `GroupBy(Rows(field=general, previous=10))`).Results[0].([]pilosa.GroupCount)
+			results := c.Query(t, "i", `GroupBy(Rows(general, previous=10))`).Results[0].([]pilosa.GroupCount)
 			test.CheckGroupBy(t, expected, results)
 		})
 
@@ -3315,7 +3378,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
 				{Group: []pilosa.FieldRow{{Field: "general", RowID: 11}}, Count: 2},
 			}
 
-			results := c.Query(t, "i", `GroupBy(Rows(field=general, previous=10), limit=1)`).Results[0].([]pilosa.GroupCount)
+			results := c.Query(t, "i", `GroupBy(Rows(general, previous=10), limit=1)`).Results[0].([]pilosa.GroupCount)
 			test.CheckGroupBy(t, expected, results)
 
 		})
@@ -3336,7 +3399,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
 				{Group: []pilosa.FieldRow{{Field: "a", RowID: 0}, {Field: "b", RowID: 1}}, Count: 1},
 			}
 
-			results := c.Query(t, "i", `GroupBy(Rows(field=a), Rows(field=b), limit=1)`).Results[0].([]pilosa.GroupCount)
+			results := c.Query(t, "i", `GroupBy(Rows(a), Rows(b), limit=1)`).Results[0].([]pilosa.GroupCount)
 			test.CheckGroupBy(t, expected, results)
 		})
 
@@ -3364,7 +3427,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
 		})
 
 		t.Run("test wrapping with previous", func(t *testing.T) {
-			results := c.Query(t, "i", `GroupBy(Rows(field=wa), Rows(field=wb), Rows(field=wc, previous=1), limit=3)`).Results[0].([]pilosa.GroupCount)
+			results := c.Query(t, "i", `GroupBy(Rows(wa), Rows(wb), Rows(wc, previous=1), limit=3)`).Results[0].([]pilosa.GroupCount)
 			expected := []pilosa.GroupCount{
 				{Group: []pilosa.FieldRow{{Field: "wa", RowID: 0}, {Field: "wb", RowID: 0}, {Field: "wc", RowID: 2}}, Count: 2},
 				{Group: []pilosa.FieldRow{{Field: "wa", RowID: 0}, {Field: "wb", RowID: 1}, {Field: "wc", RowID: 0}}, Count: 1},
@@ -3374,14 +3437,14 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
 		})
 
 		t.Run("test previous is last result", func(t *testing.T) {
-			results := c.Query(t, "i", `GroupBy(Rows(field=wa, previous=3), Rows(field=wb, previous=3), Rows(field=wc, previous=3), limit=3)`).Results[0].([]pilosa.GroupCount)
+			results := c.Query(t, "i", `GroupBy(Rows(wa, previous=3), Rows(wb, previous=3), Rows(wc, previous=3), limit=3)`).Results[0].([]pilosa.GroupCount)
 			if len(results) > 0 {
 				t.Fatalf("expected no results because previous specified last result")
 			}
 		})
 
 		t.Run("test wrapping multiple", func(t *testing.T) {
-			results := c.Query(t, "i", `GroupBy(Rows(field=wa), Rows(field=wb, previous=2), Rows(field=wc, previous=2), limit=1)`).Results[0].([]pilosa.GroupCount)
+			results := c.Query(t, "i", `GroupBy(Rows(wa), Rows(wb, previous=2), Rows(wc, previous=2), limit=1)`).Results[0].([]pilosa.GroupCount)
 			expected := []pilosa.GroupCount{
 				{Group: []pilosa.FieldRow{{Field: "wa", RowID: 1}, {Field: "wb", RowID: 0}, {Field: "wc", RowID: 0}}, Count: 1},
 			}
@@ -3405,7 +3468,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
 			{3, ShardWidth},
 		})
 		t.Run("distinct rows in different shards", func(t *testing.T) {
-			results := c.Query(t, "i", `GroupBy(Rows(field=ma), Rows(field=mb), limit=5)`).Results[0].([]pilosa.GroupCount)
+			results := c.Query(t, "i", `GroupBy(Rows(ma), Rows(mb), limit=5)`).Results[0].([]pilosa.GroupCount)
 			expected := []pilosa.GroupCount{
 				{Group: []pilosa.FieldRow{{Field: "ma", RowID: 0}, {Field: "mb", RowID: 0}}, Count: 1},
 				{Group: []pilosa.FieldRow{{Field: "ma", RowID: 0}, {Field: "mb", RowID: 2}}, Count: 1},
@@ -3417,7 +3480,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
 		})
 
 		t.Run("distinct rows in different shards with row limit", func(t *testing.T) {
-			results := c.Query(t, "i", `GroupBy(Rows(field=ma), Rows(field=mb, limit=2), limit=5)`).Results[0].([]pilosa.GroupCount)
+			results := c.Query(t, "i", `GroupBy(Rows(ma), Rows(mb, limit=2), limit=5)`).Results[0].([]pilosa.GroupCount)
 			expected := []pilosa.GroupCount{
 				{Group: []pilosa.FieldRow{{Field: "ma", RowID: 0}, {Field: "mb", RowID: 0}}, Count: 1},
 				{Group: []pilosa.FieldRow{{Field: "ma", RowID: 1}, {Field: "mb", RowID: 1}}, Count: 1},
@@ -3428,7 +3491,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
 		})
 
 		t.Run("distinct rows in different shards with column arg", func(t *testing.T) {
-			results := c.Query(t, "i", fmt.Sprintf(`GroupBy(Rows(field=ma), Rows(field=mb, column=%d), limit=5)`, ShardWidth)).Results[0].([]pilosa.GroupCount)
+			results := c.Query(t, "i", fmt.Sprintf(`GroupBy(Rows(ma), Rows(mb, column=%d), limit=5)`, ShardWidth)).Results[0].([]pilosa.GroupCount)
 			expected := []pilosa.GroupCount{
 				{Group: []pilosa.FieldRow{{Field: "ma", RowID: 1}, {Field: "mb", RowID: 1}}, Count: 1},
 				{Group: []pilosa.FieldRow{{Field: "ma", RowID: 1}, {Field: "mb", RowID: 3}}, Count: 1},
@@ -3453,7 +3516,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
 			{1, ShardWidth},
 		})
 		t.Run("same rows in different shards", func(t *testing.T) {
-			results := c.Query(t, "i", `GroupBy(Rows(field=na), Rows(field=nb))`).Results[0].([]pilosa.GroupCount)
+			results := c.Query(t, "i", `GroupBy(Rows(na), Rows(nb))`).Results[0].([]pilosa.GroupCount)
 			expected := []pilosa.GroupCount{
 				{Group: []pilosa.FieldRow{{Field: "na", RowID: 0}, {Field: "nb", RowID: 0}}, Count: 2},
 				{Group: []pilosa.FieldRow{{Field: "na", RowID: 0}, {Field: "nb", RowID: 1}}, Count: 2},
@@ -3490,11 +3553,11 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
 
 		t.Run("test wrapping with previous", func(t *testing.T) {
 			totalResults := make([]pilosa.GroupCount, 0)
-			results := c.Query(t, "i", `GroupBy(Rows(field=ppa), Rows(field=ppb), Rows(field=ppc), limit=3)`).Results[0].([]pilosa.GroupCount)
+			results := c.Query(t, "i", `GroupBy(Rows(ppa), Rows(ppb), Rows(ppc), limit=3)`).Results[0].([]pilosa.GroupCount)
 			totalResults = append(totalResults, results...)
 			for len(totalResults) < 64 {
 				lastGroup := results[len(results)-1].Group
-				query := fmt.Sprintf("GroupBy(Rows(field=ppa, previous=%d), Rows(field=ppb, previous=%d), Rows(field=ppc, previous=%d), limit=3)", lastGroup[0].RowID, lastGroup[1].RowID, lastGroup[2].RowID)
+				query := fmt.Sprintf("GroupBy(Rows(ppa, previous=%d), Rows(ppb, previous=%d), Rows(ppc, previous=%d), limit=3)", lastGroup[0].RowID, lastGroup[1].RowID, lastGroup[2].RowID)
 				results = c.Query(t, "i", query).Results[0].([]pilosa.GroupCount)
 				totalResults = append(totalResults, results...)
 			}
@@ -3537,7 +3600,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
 				{Group: []pilosa.FieldRow{{Field: "generalk", RowID: 3, RowKey: "twelve"}, {Field: "subk", RowID: 2, RowKey: "one-hundred-ten"}}, Count: 1},
 			}
 
-			results := c.Query(t, "i", `GroupBy(Rows(field="generalk"), Rows(field="subk"))`).Results[0].([]pilosa.GroupCount)
+			results := c.Query(t, "i", `GroupBy(Rows(generalk), Rows(subk))`).Results[0].([]pilosa.GroupCount)
 			test.CheckGroupBy(t, expected, results)
 
 		})
@@ -3551,7 +3614,16 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
 }
 
 func BenchmarkGroupBy(b *testing.B) {
-	c := test.MustRunCluster(b, 1)
+	c := test.MustNewCluster(b, 1)
+	var err error
+	c[0].Config.DataDir, err = ioutil.TempDir(*TempDir, "benchmarkGroupBy")
+	if err != nil {
+		b.Fatalf("getting temp dir: %v", err)
+	}
+	err = c.Start()
+	if err != nil {
+		b.Fatalf("starting cluster: %v", err)
+	}
 	defer c.Close()
 	c.CreateField(b, "i", pilosa.IndexOptions{}, "a")
 	c.CreateField(b, "i", pilosa.IndexOptions{}, "b")
@@ -3586,7 +3658,7 @@ func BenchmarkGroupBy(b *testing.B) {
 		b.ResetTimer()
 		b.ReportAllocs()
 		for i := 0; i < b.N; i++ {
-			c.Query(b, "i", `GroupBy(Rows(field=a), Rows(field=b), Rows(field=c))`)
+			c.Query(b, "i", `GroupBy(Rows(a), Rows(b), Rows(c))`)
 		}
 	})
 
@@ -3594,7 +3666,7 @@ func BenchmarkGroupBy(b *testing.B) {
 		b.ResetTimer()
 		b.ReportAllocs()
 		for i := 0; i < b.N; i++ {
-			c.Query(b, "i", `GroupBy(Rows(field=a), Rows(field=b), Rows(field=c), limit=4)`)
+			c.Query(b, "i", `GroupBy(Rows(a), Rows(b), Rows(c), limit=4)`)
 		}
 	})
 
@@ -3644,3 +3716,91 @@ func runCallTest(t *testing.T, writeQuery string, readQueries []string, indexOpt
 
 	return responses
 }
+
+func TestExecutor_Execute_Shift(t *testing.T) {
+	t.Run("Shift Bit 0", func(t *testing.T) {
+		c := test.MustRunCluster(t, 1)
+		defer c.Close()
+		hldr := test.Holder{Holder: c[0].Server.Holder()}
+		hldr.SetBit("i", "general", 10, 0)
+
+		if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10), n=1)`}); err != nil {
+			t.Fatal(err)
+		} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1}) {
+			t.Fatalf("unexpected columns: %+v", columns)
+		}
+
+		if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Shift(Row(general=10), n=1), n=1)`}); err != nil {
+			t.Fatal(err)
+		} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2}) {
+			t.Fatalf("unexpected columns: %+v", columns)
+		}
+	})
+
+	t.Run("Shift container boundary", func(t *testing.T) {
+		c := test.MustRunCluster(t, 1)
+		defer c.Close()
+		hldr := test.Holder{Holder: c[0].Server.Holder()}
+		hldr.SetBit("i", "general", 10, 65535)
+
+		if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10), n=1)`}); err != nil {
+			t.Fatal(err)
+		} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{65536}) {
+			t.Fatalf("unexpected columns: %+v", columns)
+		}
+	})
+
+	t.Run("Shift shard boundary", func(t *testing.T) {
+		c := test.MustRunCluster(t, 1)
+		defer c.Close()
+		hldr := test.Holder{Holder: c[0].Server.Holder()}
+
+		orig := []uint64{1, ShardWidth - 1, ShardWidth + 1}
+		shift1 := []uint64{2, ShardWidth, ShardWidth + 2}
+		shift2 := []uint64{3, ShardWidth + 1, ShardWidth + 3}
+
+		for _, bit := range orig {
+			hldr.SetBit("i", "general", 10, bit)
+		}
+
+		if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10), n=1)`}); err != nil {
+			t.Fatal(err)
+		} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, shift1) {
+			t.Fatalf("unexpected shift by 1: expected: %+v, but got: %+v", shift1, columns)
+		}
+
+		if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10), n=2)`}); err != nil {
+			t.Fatal(err)
+		} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, shift2) {
+			t.Fatalf("unexpected shift by 2: expected: %+v, but got: %+v", shift2, columns)
+		}
+
+		if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Shift(Row(general=10)))`}); err != nil {
+			t.Fatal(err)
+		} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, orig) {
+			t.Fatalf("unexpected shift by 0: expected: %+v, but got: %+v", orig, columns)
+		}
+	})
+
+	t.Run("Shift shard boundary no create", func(t *testing.T) {
+		c := test.MustRunCluster(t, 1)
+		defer c.Close()
+		hldr := test.Holder{Holder: c[0].Server.Holder()}
+		hldr.SetBit("i", "general", 10, ShardWidth-2) //shardwidth -1
+		hldr.SetBit("i", "general", 10, ShardWidth-1) //shardwidth
+		hldr.SetBit("i", "general", 10, ShardWidth)   //shardwidth +1
+		hldr.SetBit("i", "general", 10, ShardWidth+2) //shardwidth +3
+
+		exp := []uint64{ShardWidth - 1, ShardWidth, ShardWidth + 1, ShardWidth + 3}
+		if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10), n=1)`}); err != nil {
+			t.Fatal(err)
+		} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, exp) {
+			t.Fatalf("unexpected columns: %+v", columns)
+		}
+		if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Shift(Row(general=10), n=1), n=1)`}); err != nil {
+			t.Fatal(err)
+		} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{ShardWidth, ShardWidth + 1, ShardWidth + 2, ShardWidth + 4}) {
+			t.Fatalf("unexpected columns: \n%+v\n%+v", columns, exp)
+		}
+	})
+}

field.go

@@ -476,13 +476,21 @@ func (f *Field) saveMeta() error {
 // applyOptions configures the field based on opt.
 func (f *Field) applyOptions(opt FieldOptions) error {
 	switch opt.Type {
-	case FieldTypeSet, "":
-		f.options.Type = FieldTypeSet
+	case FieldTypeSet, FieldTypeMutex, "":
+		fldType := opt.Type
+		if fldType == "" {
+			fldType = FieldTypeSet
+		}
+		f.options.Type = fldType
 		if opt.CacheType != "" {
 			f.options.CacheType = opt.CacheType
 		}
 		if opt.CacheSize != 0 {
-			f.options.CacheSize = opt.CacheSize
+			if opt.CacheType == CacheTypeNone {
+				f.options.CacheSize = 0
+			} else {
+				f.options.CacheSize = opt.CacheSize
+			}
 		}
 		f.options.Min = 0
 		f.options.Max = 0
@@ -524,18 +532,6 @@ func (f *Field) applyOptions(opt FieldOptions) error {
 			f.Close()
 			return errors.Wrap(err, "setting time quantum")
 		}
-	case FieldTypeMutex:
-		f.options.Type = FieldTypeMutex
-		if opt.CacheType != "" {
-			f.options.CacheType = opt.CacheType
-		}
-		if opt.CacheSize != 0 {
-			f.options.CacheSize = opt.CacheSize
-		}
-		f.options.Min = 0
-		f.options.Max = 0
-		f.options.TimeQuantum = ""
-		f.options.Keys = opt.Keys
 	case FieldTypeBool:
 		f.options.Type = FieldTypeBool
 		f.options.CacheType = CacheTypeNone

field_internal_test.go

@@ -192,7 +192,7 @@ type TestField struct {
 
 // NewTestField returns a new instance of TestField d/0.
 func NewTestField(opts FieldOption) *TestField {
-	path, err := ioutil.TempDir("", "pilosa-field-")
+	path, err := ioutil.TempDir(*TempDir, "pilosa-field-")
 	if err != nil {
 		panic(err)
 	}

fragment.go

@@ -1050,7 +1050,7 @@ func (f *fragment) top(opt topOptions) ([]Pair, error) {
 		rowID, cnt := pair.ID, pair.Count
 
 		// Ignore empty rows.
-		if cnt <= 0 {
+		if cnt == 0 {
 			continue
 		}
 

fragment_internal_test.go

@@ -39,12 +39,9 @@ var (
 	// In order to generate the sample fragment file,
 	// run an import and copy PILOSA_DATA_DIR/INDEX_NAME/FRAME_NAME/0 to testdata/sample_view
 	FragmentPath = flag.String("fragment", "testdata/sample_view/0", "fragment path")
-	TempDir      = ""
-)
 
-func init() { // nolint: gochecknoinits
-	flag.StringVar(&TempDir, "temp-dir", "", "Directory in which to place temporary data (e.g. for benchmarking). Useful if you are trying to benchmark different storage configurations.")
-}
+	TempDir = flag.String("temp-dir", "", "Directory in which to place temporary data (e.g. for benchmarking). Useful if you are trying to benchmark different storage configurations.")
+)
 
 // Ensure a fragment can set a bit and retrieve it.
 func TestFragment_SetBit(t *testing.T) {
@@ -2069,7 +2066,7 @@ func BenchmarkFileWrite(b *testing.B) {
 		b.Run(fmt.Sprintf("Rows%d", numRows), func(b *testing.B) {
 			b.StopTimer()
 			for i := 0; i < b.N; i++ {
-				f, err := ioutil.TempFile(TempDir, "")
+				f, err := ioutil.TempFile(*TempDir, "")
 				if err != nil {
 					b.Fatalf("getting temp file: %v", err)
 				}
@@ -2125,7 +2122,7 @@ func (f *fragment) CleanKeep(t testing.TB) {
 
 // mustOpenFragment returns a new instance of Fragment with a temporary path.
 func mustOpenFragment(index, field, view string, shard uint64, cacheType string) *fragment {
-	file, err := ioutil.TempFile(TempDir, "pilosa-fragment-")
+	file, err := ioutil.TempFile(*TempDir, "pilosa-fragment-")
 	if err != nil {
 		panic(err)
 	}

gossip/gossip.go

@@ -22,6 +22,7 @@ import (
 	"io/ioutil"
 	"log"
 	"net"
+	"os"
 	"strconv"
 	"strings"
 	"sync"
@@ -50,8 +51,11 @@ type memberSet struct {
 
 	Logger logger.Logger
 
-	logger    *log.Logger
+	// stdLogger is only used when passed into memberlist library things that take a std library logger rather than an interface.
+	stdLogger *log.Logger
+	// logOutput is similar to stdLogger in that it's passed to memberlist things which can't take a pilosa Logger.
 	logOutput io.Writer
+
 	transport *Transport
 
 	eventReceiver *eventReceiver
@@ -151,14 +155,20 @@ func WithTransport(transport *Transport) memberSetOption {
 	}
 }
 
-// WithLogger is a functional option for providing a logger to NewMemberSet.
+// WithLogger is a functional option for providing a Go logger to NewMemberSet.
+// If the memberSet's transport is nil, this logger will be used when creating
+// one. If WithLogOutput is not used, this logger will be passed to memberlist
+// for it to use internally. This logger is not used for logging by code in this
+// (gossip) package - for that, use the WithPilosaLogger option.
 func WithLogger(logger *log.Logger) memberSetOption {
 	return func(g *memberSet) error {
-		g.logger = logger
+		g.stdLogger = logger
 		return nil
 	}
 }
 
+// WithLogOutput allows one to pass a Writer which will in turn be passed to
+// memberlist for use in logging.
 func WithLogOutput(o io.Writer) memberSetOption {
 	return func(g *memberSet) error {
 		g.logOutput = o
@@ -166,7 +176,20 @@ func WithLogOutput(o io.Writer) memberSetOption {
 	}
 }
 
-// NewMemberSet returns a new instance of GossipMemberSet based on options.
+// WithPilosaLogger allows one to configure a memberSet with a logger of their
+// choice which satisfies the pilosa logger interface.
+func WithPilosaLogger(l logger.Logger) memberSetOption {
+	return func(g *memberSet) error {
+		g.Logger = l
+		return nil
+	}
+}
+
+// NewMemberSet returns a new instance of GossipMemberSet based on options. The
+// logging options which can be passed to NewMemberSet are complicated for
+// historical reasons - please pass WithPilosaLogger, and either WithLogOutput
+// or WithLogger. If you pass WithLogOutput, be sure to also pass in a Transport
+// using WithTransport.
 func NewMemberSet(cfg Config, api *pilosa.API, options ...memberSetOption) (*memberSet, error) {
 	host := api.Node().URI.Host
 	g := &memberSet{
@@ -180,7 +203,8 @@ func NewMemberSet(cfg Config, api *pilosa.API, options ...memberSetOption) (*mem
 			return nil, errors.Wrap(err, "executing option")
 		}
 	}
-	ger := newEventReceiver(g.logger, api)
+
+	ger := newEventReceiver(g.Logger, api)
 	g.eventReceiver = ger
 
 	if g.transport == nil {
@@ -189,8 +213,16 @@ func NewMemberSet(cfg Config, api *pilosa.API, options ...memberSetOption) (*mem
 			return nil, fmt.Errorf("convert port: %s", err)
 		}
 
+		if g.stdLogger == nil {
+			if g.logOutput != nil {
+				g.stdLogger = logger.NewStandardLogger(g.logOutput).Logger()
+			} else {
+				g.stdLogger = log.New(os.Stderr, "", log.LstdFlags)
+			}
+		}
+
 		// Set up the transport.
-		transport, err := NewTransport(host, port, g.logger)
+		transport, err := NewTransport(host, port, g.stdLogger)
 		if err != nil {
 			return nil, fmt.Errorf("new tranport: %s", err)
 		}
@@ -209,14 +241,29 @@ func NewMemberSet(cfg Config, api *pilosa.API, options ...memberSetOption) (*mem
 		}
 	}
 
+	////////////////////
 	// memberlist config
 	conf := memberlist.DefaultWANConfig()
 	conf.Transport = g.transport.net
 	conf.Name = api.Node().ID
 	conf.BindAddr = api.Node().URI.Host
 	conf.BindPort = port
-	conf.AdvertisePort = port
-	conf.AdvertiseAddr = hostToIP(api.Node().URI.Host)
+	// AdvertisePort
+	if cfg.AdvertisePort != "" {
+		if p, err := strconv.Atoi(cfg.Port); err != nil {
+			return nil, fmt.Errorf("convert advertise port: %s", err)
+		} else {
+			conf.AdvertisePort = p
+		}
+	} else {
+		conf.AdvertisePort = port
+	}
+	// AdvertiseHost
+	if cfg.AdvertiseHost != "" {
+		conf.AdvertiseAddr = cfg.AdvertiseHost
+	} else {
+		conf.AdvertiseAddr = hostToIP(api.Node().URI.Host)
+	}
 	//
 	conf.TCPTimeout = time.Duration(cfg.StreamTimeout)
 	conf.SuspicionMult = cfg.SuspicionMult
@@ -233,7 +280,7 @@ func NewMemberSet(cfg Config, api *pilosa.API, options ...memberSetOption) (*mem
 	if g.logOutput != nil {
 		conf.LogOutput = g.logOutput
 	} else {
-		conf.Logger = g.logger
+		conf.Logger = g.stdLogger
 	}
 
 	g.config = &config{
@@ -318,11 +365,11 @@ type eventReceiver struct {
 	ch   chan memberlist.NodeEvent
 	papi *pilosa.API
 
-	logger *log.Logger
+	logger logger.Logger
 }
 
 // newEventReceiver returns a new instance of GossipEventReceiver.
-func newEventReceiver(logger *log.Logger, papi *pilosa.API) *eventReceiver {
+func newEventReceiver(logger logger.Logger, papi *pilosa.API) *eventReceiver {
 	ger := &eventReceiver{
 		ch:     make(chan memberlist.NodeEvent, 1),
 		logger: logger,
@@ -468,7 +515,7 @@ func newTransport(conf *memberlist.Config) (*memberlist.NetTransport, error) {
 
 	nt, err := makeNetRetry(limit)
 	if err != nil {
-		return nil, fmt.Errorf("Could not set up network transport: %v", err)
+		return nil, errors.Wrap(err, "could not set up network transport")
 	}
 
 	return nt, nil
@@ -477,7 +524,16 @@ func newTransport(conf *memberlist.Config) (*memberlist.NetTransport, error) {
 // Config holds toml-friendly memberlist configuration.
 type Config struct {
 	// Port indicates the port to which pilosa should bind for internal state sharing.
-	Port  string   `toml:"port"`
+	Port string `toml:"port"`
+
+	// AdvertiseHost is the hostname or IP other nodes should use to connect to
+	// this host. If left blank, the value for Host will be used. This is useful
+	// in some proxy and NAT scenarios.
+	AdvertiseHost string `toml:"advertise-host"`
+	// AdvertisePort is the port other nodes will use to connect to this one.
+	// Behaves like AdvertiseHost.
+	AdvertisePort string `toml:"advertise-port"`
+
 	Seeds []string `toml:"seeds"`
 	Key   string   `toml:"key"`
 	// StreamTimeout is the timeout for establishing a stream connection with

holder_internal_test.go

@@ -46,7 +46,7 @@ func (h *tHolder) Reopen() error {
 }
 
 func newHolder() *tHolder {
-	path, err := ioutil.TempDir("", "pilosa-")
+	path, err := ioutil.TempDir(*TempDir, "pilosa-")
 	if err != nil {
 		panic(err)
 	}

http/client.go

@@ -1015,7 +1015,7 @@ func (c *InternalClient) executeRequest(req *http.Request) (*http.Response, erro
 		if resp != nil {
 			resp.Body.Close()
 		}
-		return nil, errors.Wrap(err, "executing request")
+		return nil, errors.Wrap(err, "getting response")
 	}
 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
 		defer resp.Body.Close()

http/handler.go

@@ -584,11 +584,11 @@ func validateOptions(data map[string]interface{}, validIndexOptions []string) er
 			}
 			for kk, vv := range options {
 				if !foundItem(validIndexOptions, kk) {
-					return fmt.Errorf("Unknown key: %v:%v", kk, vv)
+					return fmt.Errorf("unknown key: %v:%v", kk, vv)
 				}
 			}
 		default:
-			return fmt.Errorf("Unknown key: %v:%v", k, v)
+			return fmt.Errorf("unknown key: %v:%v", k, v)
 		}
 	}
 	return nil

http/handler_internal_test.go

@@ -35,8 +35,8 @@ func TestPostIndexRequestUnmarshalJSON(t *testing.T) {
 		{json: `{"options": {"trackExistence": false}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{TrackExistence: false}}},
 		{json: `{"options": {"keys": true}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{Keys: true, TrackExistence: true}}},
 		{json: `{"options": 4}`, err: "options is not map[string]interface{}"},
-		{json: `{"option": {}}`, err: "Unknown key: option:map[]"},
-		{json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"},
+		{json: `{"option": {}}`, err: "unknown key: option:map[]"},
+		{json: `{"options": {"badKey": "test"}}`, err: "unknown key: badKey:test"},
 	}
 	for _, test := range tests {
 		actual := &postIndexRequest{}

index_internal_test.go

@@ -21,7 +21,7 @@ import (
 
 // mustOpenIndex returns a new, opened index at a temporary path. Panic on error.
 func mustOpenIndex(opt IndexOptions) *Index {
-	path, err := ioutil.TempDir("", "pilosa-index-")
+	path, err := ioutil.TempDir(*TempDir, "pilosa-index-")
 	if err != nil {
 		panic(err)
 	}

lru/lru.go

@@ -83,7 +83,7 @@ func (c *Cache) Get(key Key) (value interface{}, ok bool) {
 }
 
 // remove removes the provided key from the cache.
-func (c *Cache) remove(key Key) { // nolint: megacheck
+func (c *Cache) remove(key Key) { // nolint: staticcheck
 	if c.cache == nil {
 		return
 	}
@@ -121,7 +121,7 @@ func (c *Cache) Len() int {
 }
 
 // clear purges all stored items from the cache.
-func (c *Cache) clear() { // nolint: megacheck
+func (c *Cache) clear() { // nolint: staticcheck
 	if c.OnEvicted != nil {
 		for _, e := range c.cache {
 			kv := e.Value.(*entry)

pilosa.go

@@ -166,7 +166,6 @@ func stringSlicesAreEqual(a, b []string) bool {
 func AddressWithDefaults(addr string) (*URI, error) {
 	if addr == "" {
 		return defaultURI(), nil
-	} else {
-		return NewURIFromAddress(addr)
 	}
+	return NewURIFromAddress(addr)
 }

pql/ast.go

@@ -259,7 +259,7 @@ func (c *Call) FieldArg() (string, error) {
 			return arg, nil
 		}
 	}
-	return "", fmt.Errorf("No field argument specified")
+	return "", fmt.Errorf("no field argument specified")
 }
 
 func IsReservedArg(name string) bool {

pql/pql.peg

@@ -13,6 +13,7 @@ Call <-  'Set' {p.startCall("Set")} open col comma args (comma timestamp)? close
        / 'ClearRow' {p.startCall("ClearRow")} open arg close {p.endCall()}
        / 'Store' {p.startCall("Store")} open Call comma arg close {p.endCall()}
        / 'TopN' {p.startCall("TopN")} open posfield (comma allargs)? close {p.endCall()}
+       / 'Rows' {p.startCall("Rows")} open posfield (comma allargs)? close {p.endCall()}
        / < IDENT > { p.startCall(buffer[begin:end] ) } open allargs comma? close { p.endCall() }
 allargs <- Call (comma Call)* (comma args)? / args / sp
 args <- arg (comma args)? sp

roaring/roaring.go

@@ -52,16 +52,20 @@ const (
 	// bitmapN is the number of values in a container.bitmap.
 	bitmapN = (1 << 16) / 64
 
-	//containerArray indicates a container of bit position values
+	// containerArray indicates a container of bit position values
 	containerArray = byte(1)
 
-	//containerBitmap indicates a container of bits packed in a uint64 array block
+	// containerBitmap indicates a container of bits packed in a uint64 array block
 	containerBitmap = byte(2)
 
-	//containerRun  indicates a container of run encoded bits
+	// containerRun  indicates a container of run encoded bits
 	containerRun = byte(3)
 
 	maxContainerVal = 0xffff
+
+	// maxContainerKey is the key representing the last container in a full row.
+	// It is the full bitmap space (2^64) divided by container width (2^16).
+	maxContainerKey = (1 << 48) - 1
 )
 
 type Containers interface {
@@ -163,9 +167,8 @@ func (b *Bitmap) Add(a ...uint64) (changed bool, err error) {
 		}
 
 		// Apply to the in-memory bitmap.
-		if op.apply(b) {
+		if b.DirectAdd(v) {
 			changed = true
-
 		}
 	}
 
@@ -233,6 +236,18 @@ func (b *Bitmap) Count() (n uint64) {
 	return b.Containers.Count()
 }
 
+// Size returns the number of bytes required for the bitmap.
+func (b *Bitmap) Size() int {
+	numbytes := 0
+	citer, _ := b.Containers.Iterator(0)
+	for citer.Next() {
+		_, c := citer.Value()
+		numbytes += c.size()
+
+	}
+	return numbytes
+}
+
 // CountRange returns the number of bits set between [start, end).
 func (b *Bitmap) CountRange(start, end uint64) (n uint64) {
 	if b.Containers.Size() == 0 {
@@ -742,6 +757,38 @@ func (b *Bitmap) Xor(other *Bitmap) *Bitmap {
 	return output
 }
 
+// Shift shifts the contents of b by 1.
+func (b *Bitmap) Shift(n int) (*Bitmap, error) {
+	if n != 1 {
+		return nil, errors.New("cannot shift by a value other than 1")
+	}
+	output := NewBitmap()
+	iiter, _ := b.Containers.Iterator(0)
+	lastCarry := false
+	lastKey := uint64(0)
+	for iiter.Next() {
+		ki, ci := iiter.Value()
+		o, carry := shift(ci)
+		if lastCarry {
+			o.add(0)
+		}
+		if o.n > 0 {
+			output.Containers.Put(ki, o)
+		}
+		lastCarry = carry
+		lastKey = ki
+	}
+	// As long as the carry wasn't from the max container,
+	// append a new container and add the carried bit.
+	if lastCarry && lastKey != maxContainerKey {
+		extra := NewContainer()
+		extra.add(0)
+		output.Containers.Put(lastKey+1, extra)
+	}
+
+	return output, nil
+}
+
 // removeEmptyContainers deletes all containers that have a count of zero.
 func (b *Bitmap) removeEmptyContainers() {
 	citer, _ := b.Containers.Iterator(0)
@@ -3350,6 +3397,88 @@ func xorBitmapBitmap(a, b *Container) *Container {
 	return output
 }
 
+// shift() shifts the contents of c by one. It returns
+// the new container and a bool indicating whether a
+// carry bit was shifted out.
+func shift(c *Container) (*Container, bool) {
+	if c.isArray() {
+		return shiftArray(c)
+	} else if c.isRun() {
+		return shiftRun(c)
+	}
+	return shiftBitmap(c)
+}
+
+// shiftArray is an array-specific implementation of shift().
+func shiftArray(a *Container) (*Container, bool) {
+	statsHit("shift/Array")
+	carry := false
+	output := &Container{containerType: containerArray}
+	output.array = make([]uint16, len(a.array))
+	output.array = output.array[:0]
+	output.n = a.n
+	for _, v := range a.array {
+		if v+1 == 0 { // overflow
+			carry = true
+			output.n -= 1
+		} else {
+			output.array = append(output.array, v+1)
+		}
+	}
+	return output, carry
+}
+
+// shiftBitmap is a bitmap-specific implementation of shift().
+func shiftBitmap(a *Container) (*Container, bool) {
+	statsHit("shift/Bitmap")
+	carry := false
+	output := &Container{containerType: containerBitmap}
+	output.bitmap = make([]uint64, len(a.bitmap))
+	output.bitmap = output.bitmap[:0]
+	output.n = a.n
+	lastCarry := false
+	for _, v := range a.bitmap {
+		carry = (v & (1 << 63)) != 0
+		v = v << 1
+		if lastCarry {
+			v |= 1
+		}
+		output.bitmap = append(output.bitmap, v)
+		lastCarry = carry
+	}
+	if carry {
+		output.n -= 1
+	}
+	return output, carry
+}
+
+// shiftRun is a run-specific implementation of shift().
+func shiftRun(a *Container) (*Container, bool) {
+	statsHit("shift/Run")
+	carry := false
+	output := &Container{containerType: containerRun}
+	output.runs = make([]interval16, len(a.runs))
+	output.runs = output.runs[:0]
+	for _, v := range a.runs {
+		if v.start+1 == 0 { // final run was 1 bit on container edge
+			carry = true
+			output.n -= 1
+			break
+		} else if v.last+1 == 0 { // final run ends on container edge
+			v.start += 1
+			carry = true
+			output.n -= 1
+		} else {
+			v.start += 1
+			v.last += 1
+			carry = false
+		}
+		output.runs = append(output.runs, v)
+	}
+
+	return output, carry
+}
+
 // opType represents a type of operation.
 type opType uint8
 
@@ -3869,7 +3998,7 @@ func readOfficialHeader(buf []byte) (size uint32, containerTyper func(index uint
 
 	header = pos
 	if size > (1 << 16) {
-		err = fmt.Errorf("It is logically impossible to have more than (1<<16) containers.")
+		err = fmt.Errorf("it is logically impossible to have more than (1<<16) containers")
 		return size, containerTyper, header, pos, haveRuns, err
 	}
 

roaring/roaring_helpers_test.go

@@ -106,6 +106,24 @@ func bitmapFirstBitSet() []uint64 {
 	return bitmap
 }
 
+func bitmapSecondBitSet() []uint64 {
+	bitmap := make([]uint64, bitmapN)
+	bitmap[0] = 0x0000000000000002
+	return bitmap
+}
+
+func bitmapLastBitFirstRowSet() []uint64 {
+	bitmap := make([]uint64, bitmapN)
+	bitmap[0] = 0x8000000000000000
+	return bitmap
+}
+
+func bitmapFirstBitSecoundRowSet() []uint64 {
+	bitmap := make([]uint64, bitmapN)
+	bitmap[1] = 0x0000000000000001
+	return bitmap
+}
+
 func bitmapLastBitSet() []uint64 {
 	bitmap := make([]uint64, bitmapN)
 	bitmap[bitmapN-1] = 0x8000000000000000

roaring/roaring_internal_test.go

@@ -3224,11 +3224,11 @@ func TestContainerCombinations(t *testing.T) {
 
 //func getFunc(func(a, b *container) *container, m, n *container) *container {
 func runContainerFunc(f interface{}, c ...*Container) *Container {
-	switch f.(type) {
+	switch f := f.(type) {
 	case func(*Container) *Container:
-		return f.(func(*Container) *Container)(c[0])
+		return f(c[0])
 	case func(*Container, *Container) *Container:
-		return f.(func(a, b *Container) *Container)(c[0], c[1])
+		return f(c[0], c[1])
 	}
 	return nil
 }
@@ -3317,3 +3317,123 @@ func TestEquals(t *testing.T) {
 	}
 }
 */
+func TestShiftArray(t *testing.T) {
+	a := &Container{
+		containerType: containerArray,
+	}
+	tests := []struct {
+		array []uint16
+		exp   []uint16
+	}{
+		{
+			array: []uint16{1},
+			exp:   []uint16{2},
+		},
+		{
+			array: []uint16{},
+			exp:   []uint16{},
+		},
+		{
+			array: []uint16{1, 2, 3, 4, 5, 11, 12},
+			exp:   []uint16{2, 3, 4, 5, 6, 12, 13},
+		},
+		{
+			array: []uint16{65535},
+			exp:   []uint16{},
+		},
+	}
+
+	for i, test := range tests {
+		a.array = test.array
+		a.n = int32(len(a.array))
+		ret1, _ := shift(a)      // test generic shift function
+		ret2, _ := shiftArray(a) // test array-specific shift function
+		if !reflect.DeepEqual(ret1.array, test.exp) {
+			t.Fatalf("test #%v shift() expected %v, but got %v", i, test.exp, ret1.array)
+		} else if !reflect.DeepEqual(ret2.array, test.exp) {
+			t.Fatalf("test #%v shiftArray() expected %v, but got %v", i, test.exp, ret2.array)
+		}
+	}
+}
+
+func TestShiftBitmap(t *testing.T) {
+	a := &Container{
+		containerType: containerBitmap,
+	}
+	tests := []struct {
+		bitmap []uint64
+		exp    []uint64
+	}{
+		{
+			bitmap: bitmapFirstBitSet(),
+			exp:    bitmapSecondBitSet(),
+		},
+		{
+			bitmap: bitmapLastBitSet(),
+			exp:    bitmapEmpty(),
+		},
+		{
+			bitmap: bitmapLastBitFirstRowSet(),
+			exp:    bitmapFirstBitSecoundRowSet(),
+		},
+	}
+
+	for i, test := range tests {
+		a.bitmap = test.bitmap
+		a.n = 1
+		ret1, _ := shift(a)       // test generic shift function
+		ret2, _ := shiftBitmap(a) // test bitmap-specific shift function
+		if !reflect.DeepEqual(ret1.bitmap, test.exp) {
+			t.Fatalf("test #%v shift() expected %v, but got %v", i, test.exp, ret1.bitmap)
+		} else if !reflect.DeepEqual(ret2.bitmap, test.exp) {
+			t.Fatalf("test #%v shiftBitmap() expected %v, but got %v", i, test.exp, ret2.bitmap)
+		}
+	}
+}
+func TestShiftRun(t *testing.T) {
+	a := &Container{
+		containerType: containerRun,
+	}
+
+	tests := []struct {
+		runs  []interval16
+		n     int32
+		en    int32
+		exp   []interval16
+		carry bool
+	}{
+		{
+			runs:  []interval16{{start: 5, last: 10}},
+			n:     5,
+			en:    5,
+			exp:   []interval16{{start: 6, last: 11}},
+			carry: false,
+		},
+		{
+			runs:  []interval16{{start: 5, last: 65535}},
+			n:     65530,
+			en:    65529,
+			exp:   []interval16{{start: 6, last: 65535}},
+			carry: true,
+		},
+		{
+			runs:  []interval16{{start: 65535, last: 65535}},
+			n:     1,
+			en:    0,
+			exp:   []interval16{},
+			carry: true,
+		},
+	}
+
+	for i, test := range tests {
+		a.runs = test.runs
+		a.n = test.n
+		ret1, c1 := shift(a)    // test generic shift function
+		ret2, c2 := shiftRun(a) // test run-specific shift function
+		if !reflect.DeepEqual(ret1.runs, test.exp) && c1 == test.carry && ret1.n == test.en {
+			t.Fatalf("test #%v shift() expected %v, but got %v %d", i, test.exp, ret1.runs, ret1.n)
+		} else if !reflect.DeepEqual(ret2.runs, test.exp) && c2 == test.carry && ret2.n == test.en {
+			t.Fatalf("test #%v shiftRun() expected %v, but got %v %d", i, test.exp, ret2.runs, ret2.n)
+		}
+	}
+}

roaring/roaring_test.go

@@ -37,6 +37,29 @@ func TestContainerCount(t *testing.T) {
 		t.Fatalf("Count != CountRange\n")
 	}
 }
+func TestSize(t *testing.T) {
+	//array
+	a := roaring.NewFileBitmap(0, 65535, 131072)
+	if a.Size() != 6 {
+		t.Fatalf("Size in bytes incorrect \n")
+	}
+
+	//bitmap
+	b := roaring.NewFileBitmap()
+	for i := uint64(0); i <= 4096; i++ {
+		b.DirectAdd(i)
+	}
+
+	if b.Size() != 8192 {
+		t.Fatalf("Size in bytes incorrect \n")
+	}
+	//convert to rle
+	b.Optimize()
+	//rle
+	if b.Size() != 6 {
+		t.Fatalf("Size in bytes incorrect \n")
+	}
+}
 
 func TestCountRange(t *testing.T) {
 	tests := []struct {
@@ -962,6 +985,18 @@ func TestBitmap_IntersectionCount_Mixed(t *testing.T) {
 	}
 }
 
+func TestBitmap_Shift(t *testing.T) {
+	var max uint64 = math.MaxUint64
+	bm1 := roaring.NewFileBitmap(0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 65536, max)
+	bm2 := roaring.NewFileBitmap(1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 65537)
+
+	if got, err := bm1.Shift(1); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(got.Slice(), bm2.Slice()) {
+		t.Fatalf("unexpected bitmap: expected %v, but got %v", bm2.Slice(), got.Slice())
+	}
+}
+
 func TestBitmap_Quick_Array1(t *testing.T)     { testBitmapQuick(t, 1000, 1000, 2000) }
 func TestBitmap_Quick_Array2(t *testing.T)     { testBitmapQuick(t, 10000, 0, 1000) }
 func TestBitmap_Quick_Bitmap1(t *testing.T)    { testBitmapQuick(t, 10000, 0, 10000) }
@@ -1498,6 +1533,41 @@ func BenchmarkSliceDescending(b *testing.B) {
 		for col := uint64(pilosa.ShardWidth); col > uint64(0); col-- {
 			bm.Add(col)
 		}
+		bm.Add(0)
+	}
+}
+
+func BenchmarkSliceAscendingStriped(b *testing.B) {
+	for n := 0; n < b.N; n++ {
+		bm := roaring.NewFileBitmap()
+		l := uint64(pilosa.ShardWidth / 8)
+		for col := uint64(0); col < l; col++ {
+			bm.Add(l*0 + col)
+			bm.Add(l*1 + col)
+			bm.Add(l*2 + col)
+			bm.Add(l*3 + col)
+			bm.Add(l*4 + col)
+			bm.Add(l*5 + col)
+			bm.Add(l*6 + col)
+			bm.Add(l*7 + col)
+		}
+	}
+}
+
+func BenchmarkSliceDescendingStriped(b *testing.B) {
+	for n := 0; n < b.N; n++ {
+		bm := roaring.NewFileBitmap()
+		l := uint64(pilosa.ShardWidth / 8)
+		for col := uint64(l); col < l+1; col-- {
+			bm.Add(l*7 + col)
+			bm.Add(l*6 + col)
+			bm.Add(l*5 + col)
+			bm.Add(l*4 + col)
+			bm.Add(l*3 + col)
+			bm.Add(l*2 + col)
+			bm.Add(l*1 + col)
+			bm.Add(l*0 + col)
+		}
 	}
 }
 

row.go

@@ -19,6 +19,7 @@ import (
 	"sort"
 
 	"github.com/pilosa/pilosa/roaring"
+	"github.com/pkg/errors"
 )
 
 // Row is a set of integers (the associated columns), and attributes which are
@@ -167,6 +168,32 @@ func (r *Row) Difference(other *Row) *Row {
 	return &Row{segments: segments}
 }
 
+// Shift returns the bitwise shift of r by n bits.
+// Currently only positive shift values are supported.
+func (r *Row) Shift(n int64) (*Row, error) {
+	if n < 0 {
+		return nil, errors.New("cannot shift by negative values")
+	} else if n == 0 {
+		return r, nil
+	}
+
+	work := r
+	var segments []rowSegment
+	for i := int64(0); i < n; i++ {
+		segments = segments[:0]
+		for _, segment := range work.segments {
+			shifted, err := segment.Shift()
+			if err != nil {
+				return nil, errors.Wrap(err, "shifting row segment")
+			}
+			segments = append(segments, *shifted)
+		}
+		work = &Row{segments: segments}
+	}
+
+	return work, nil
+}
+
 // SetBit sets the i-th column of the row.
 func (r *Row) SetBit(i uint64) (changed bool) {
 	return r.createSegmentIfNotExists(i / ShardWidth).SetBit(i)
@@ -341,6 +368,21 @@ func (s *rowSegment) Xor(other *rowSegment) *rowSegment {
 	}
 }
 
+// Shift returns s shifted by 1 bit.
+func (s *rowSegment) Shift() (*rowSegment, error) {
+	//TODO deal with overflow
+	data, err := s.data.Shift(1)
+	if err != nil {
+		return nil, errors.Wrap(err, "shifting roaring data")
+	}
+
+	return &rowSegment{
+		data:  *data,
+		shard: s.shard,
+		n:     data.Count(),
+	}, nil
+}
+
 // SetBit sets the i-th column of the row.
 func (s *rowSegment) SetBit(i uint64) (changed bool) {
 	s.ensureWritable()

server.go

@@ -487,7 +487,7 @@ func (s *Server) receiveMessage(m Message) error {
 	case *CreateShardMessage:
 		f := s.holder.Field(obj.Index, obj.Field)
 		if f == nil {
-			return fmt.Errorf("Local field not found: %s/%s", obj.Index, obj.Field)
+			return fmt.Errorf("local field not found: %s/%s", obj.Index, obj.Field)
 		}
 		if err := f.AddRemoteAvailableShards(roaring.NewBitmap(obj.Shard)); err != nil {
 			return errors.Wrap(err, "adding remote available shards")
@@ -505,7 +505,7 @@ func (s *Server) receiveMessage(m Message) error {
 	case *CreateFieldMessage:
 		idx := s.holder.Index(obj.Index)
 		if idx == nil {
-			return fmt.Errorf("Local Index not found: %s", obj.Index)
+			return fmt.Errorf("local index not found: %s", obj.Index)
 		}
 		opt := obj.Meta
 		_, err := idx.createField(obj.Field, *opt)
@@ -525,7 +525,7 @@ func (s *Server) receiveMessage(m Message) error {
 	case *CreateViewMessage:
 		f := s.holder.Field(obj.Index, obj.Field)
 		if f == nil {
-			return fmt.Errorf("Local Field not found: %s", obj.Field)
+			return fmt.Errorf("local field not found: %s", obj.Field)
 		}
 		_, _, err := f.createViewIfNotExistsBase(obj.View)
 		if err != nil {
@@ -534,7 +534,7 @@ func (s *Server) receiveMessage(m Message) error {
 	case *DeleteViewMessage:
 		f := s.holder.Field(obj.Index, obj.Field)
 		if f == nil {
-			return fmt.Errorf("Local Field not found: %s", obj.Field)
+			return fmt.Errorf("local field not found: %s", obj.Field)
 		}
 		err := f.deleteView(obj.View)
 		if err != nil {

server/cluster_test.go

@@ -399,7 +399,7 @@ func TestClusterResize_RemoveNode(t *testing.T) {
 		nodeID := mustNodeID(m0.URL())
 		resp := test.MustDo("POST", m0.URL()+fmt.Sprintf("/cluster/resize/remove-node"), fmt.Sprintf(`{"id": "%s"}`, nodeID))
 
-		expBody := "removing node: calling node leave: coordinator cannot be removed; first, make a different node the new coordinator."
+		expBody := "removing node: calling node leave: coordinator cannot be removed; first, make a different node the new coordinator"
 		if resp.StatusCode != http.StatusInternalServerError {
 			t.Fatalf("expected StatusCode %d but got %d", http.StatusInternalServerError, resp.StatusCode)
 		} else if strings.TrimSpace(resp.Body) != expBody {

server/config.go

@@ -15,10 +15,17 @@
 package server
 
 import (
+	"context"
+	"fmt"
+	"log"
+	"net"
+	"strconv"
+	"strings"
 	"time"
 
 	"github.com/pilosa/pilosa/gossip"
 	"github.com/pilosa/pilosa/toml"
+	"github.com/pkg/errors"
 	"github.com/uber/jaeger-client-go"
 )
 
@@ -37,9 +44,15 @@ type Config struct {
 	// DataDir is the directory where Pilosa stores both indexed data and
 	// running state such as cluster topology information.
 	DataDir string `toml:"data-dir"`
+
 	// Bind is the host:port on which Pilosa will listen.
 	Bind string `toml:"bind"`
 
+	// Advertise is the address advertised by the server to other nodes
+	// in the cluster. It should be reachable by all other nodes and should
+	// route to an interface that Bind is listening on.
+	Advertise string `toml:"advertise"`
+
 	// MaxWritesPerRequest limits the number of mutating commands that can be in
 	// a single request to the server. This includes Set, Clear,
 	// SetRowAttrs & SetColumnAttrs.
@@ -110,22 +123,17 @@ func NewConfig() *Config {
 		DataDir:             "~/.pilosa",
 		Bind:                ":10101",
 		MaxWritesPerRequest: 5000,
-		// LogPath: "",
-		// Verbose: false,
-		TLS: TLSConfig{},
+		TLS:                 TLSConfig{},
 	}
 
 	// Cluster config.
 	c.Cluster.Disabled = false
-	// c.Cluster.Coordinator = false
 	c.Cluster.ReplicaN = 1
 	c.Cluster.Hosts = []string{}
 	c.Cluster.LongQueryTime = toml.Duration(time.Minute)
 
 	// Gossip config.
 	c.Gossip.Port = "14000"
-	// c.Gossip.Seeds = []string{}
-	// c.Gossip.Key = ""
 	c.Gossip.StreamTimeout = toml.Duration(10 * time.Second)
 	c.Gossip.SuspicionMult = 4
 	c.Gossip.PushPullInterval = toml.Duration(30 * time.Second)
@@ -140,7 +148,6 @@ func NewConfig() *Config {
 
 	// Metric config.
 	c.Metric.Service = "none"
-	// c.Metric.Host = ""
 	c.Metric.PollInterval = toml.Duration(0 * time.Minute)
 	c.Metric.Diagnostics = true
 
@@ -150,3 +157,197 @@ func NewConfig() *Config {
 
 	return c
 }
+
+// validateAddrs controls the address fields in the Config object
+// and fills in any blanks.
+// The addresses fields must be guaranteed by the caller to either be
+// completely empty, or have both a host part and a port part
+// separated by a colon. In the latter case either can be empty to
+// indicate it's left unspecified.
+func (cfg *Config) validateAddrs(ctx context.Context) error {
+	// Validate the advertise address.
+	advScheme, advHost, advPort, err := validateAdvertiseAddr(ctx, cfg.Advertise, cfg.Bind)
+	if err != nil {
+		return errors.Wrapf(err, "validating advertise address")
+	}
+	cfg.Advertise = schemeHostPortString(advScheme, advHost, advPort)
+
+	// Validate the listen address.
+	listenScheme, listenHost, listenPort, err := validateListenAddr(ctx, cfg.Bind)
+	if err != nil {
+		return errors.Wrap(err, "validating listen address")
+	}
+	cfg.Bind = schemeHostPortString(listenScheme, listenHost, listenPort)
+
+	return nil
+}
+
+// validateAdvertiseAddr validates and normalizes an address accessible
+// Ensures that if the "host" part is empty, it gets filled in with
+// the configured listen address if any, otherwise it makes a best
+// guess at the outbound IP address.
+// Returns scheme, host, port as strings.
+func validateAdvertiseAddr(ctx context.Context, advAddr, listenAddr string) (string, string, string, error) {
+	listenScheme, listenHost, listenPort, err := splitAddr(listenAddr)
+	if err != nil {
+		return "", "", "", errors.Wrap(err, "getting listen address")
+	}
+
+	advScheme, advHostPort := splitScheme(advAddr)
+	advHost, advPort := "", ""
+	if advHostPort != "" {
+		var err error
+		advHost, advPort, err = net.SplitHostPort(advHostPort)
+		if err != nil {
+			return "", "", "", errors.Wrapf(err, "splitting host port: %s", advHostPort)
+		}
+	}
+	// If no advertise scheme was specified, use the one from
+	// the listen address.
+	if advScheme == "" {
+		advScheme = listenScheme
+	}
+	// If there was no port number, reuse the one from the listen
+	// address.
+	if advPort == "" || advPort == "0" {
+		advPort = listenPort
+	}
+	// Resolve non-numeric to numeric.
+	portNumber, err := net.DefaultResolver.LookupPort(ctx, "tcp", advPort)
+	if err != nil {
+		return "", "", "", errors.Wrapf(err, "looking up non-numeric port: %v", advPort)
+	}
+	advPort = strconv.Itoa(portNumber)
+
+	// If the advertise host is empty, then we have two cases.
+	if advHost == "" {
+		if listenHost == "0.0.0.0" {
+			advHost = outboundIP().String()
+		} else {
+			advHost = listenHost
+		}
+	}
+	return advScheme, advHost, advPort, nil
+}
+
+// outboundIP gets the preferred outbound ip of this machine.
+func outboundIP() net.IP {
+	// This is not actually making a connection to 8.8.8.8.
+	// net.Dial() selects the IP address that would be used
+	// if an actual connection to 8.8.8.8 were made, so this
+	// choice of address is just meant to ensure that an
+	// external address is returned (as opposed to a local
+	// address like 127.0.0.1).
+	conn, err := net.Dial("udp", "8.8.8.8:80")
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer conn.Close()
+
+	localAddr := conn.LocalAddr().(*net.UDPAddr)
+
+	return localAddr.IP
+}
+
+// validateListenAddr validates and normalizes an address suitable for
+// use with net.Listen(). This accepts an empty "host" part to signify
+// the default (localhost) should be used. Rresolves host names to IP
+// addresses.
+// Returns scheme, host, port as strings.
+func validateListenAddr(ctx context.Context, addr string) (string, string, string, error) {
+	scheme, host, port, err := splitAddr(addr)
+	if err != nil {
+		return "", "", "", errors.Wrap(err, "getting listen address")
+	}
+	rHost, rPort, err := resolveAddr(ctx, host, port)
+	if err != nil {
+		return "", "", "", errors.Wrap(err, "resolving address")
+	}
+	return scheme, rHost, rPort, nil
+}
+
+// splitScheme returns two strings: the scheme and the hostPort.
+func splitScheme(addr string) (string, string) {
+	parts := strings.SplitN(addr, "://", 2)
+	if len(parts) == 1 {
+		return "", addr
+	}
+	return parts[0], parts[1]
+}
+
+func schemeHostPortString(scheme, host, port string) string {
+	var s string
+	if scheme != "" {
+		s += fmt.Sprintf("%s://", scheme)
+	}
+	return s + net.JoinHostPort(host, port)
+}
+
+// splitAddr returns scheme, host, port as strings.
+func splitAddr(addr string) (string, string, string, error) {
+	scheme, hostPort := splitScheme(addr)
+	host, port := "", ""
+	if hostPort != "" {
+		var err error
+		host, port, err = net.SplitHostPort(hostPort)
+		if err != nil {
+			return "", "", "", errors.Wrapf(err, "splitting host port: %s", hostPort)
+		}
+	}
+	// It's not ideal to have a default here, but the alterative
+	// results in a port of 0, which causes Pilosa to listen on
+	// a random port.
+	if port == "" {
+		port = "10101"
+	}
+	return scheme, host, port, nil
+}
+
+// resolveAddr resolves non-numeric addresses to numeric (IP, port) addresses.
+func resolveAddr(ctx context.Context, host, port string) (string, string, error) {
+	resolver := net.DefaultResolver
+
+	// Resolve the port number. This may translate service names
+	// e.g. "postgresql" to a numeric value.
+	portNumber, err := resolver.LookupPort(ctx, "tcp", port)
+	if err != nil {
+		return "", "", errors.Wrapf(err, "resolving up port: %v", port)
+	}
+	port = strconv.Itoa(portNumber)
+
+	// Resolve the address.
+	if host == "" || host == "localhost" {
+		return host, port, nil
+	}
+
+	addr, err := lookupAddr(ctx, resolver, host)
+	if err != nil {
+		return "", "", errors.Wrap(err, "looking up address")
+	}
+	return addr, port, nil
+}
+
+// lookupAddr resolves the given address/host to an IP address. If
+// multiple addresses are resolved, it returns the first IPv4 address
+// available if there is one, otherwise the first address.
+func lookupAddr(ctx context.Context, resolver *net.Resolver, host string) (string, error) {
+	// Resolve the IP address or hostname to an IP address.
+	addrs, err := resolver.LookupIPAddr(ctx, host)
+	if err != nil {
+		return "", errors.Wrap(err, "looking up IP addresses")
+	}
+	if len(addrs) == 0 {
+		return "", fmt.Errorf("cannot resolve %q to an address", host)
+	}
+
+	// LookupIPAddr() can return a mix of IPv6 and IPv4
+	// addresses. Return the first IPv4 address if possible.
+	for _, addr := range addrs {
+		if ip := addr.IP.To4(); ip != nil {
+			return ip.String(), nil
+		}
+	}
+
+	// No IPv4 address, return the first resolved address instead.
+	return addrs[0].String(), nil
+}

server/config_internal_test.go

@@ -0,0 +1,153 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package server
+
+import (
+	"context"
+	"net"
+	"os"
+	"strings"
+	"testing"
+)
+
+type addrs struct{ bind, advertise string }
+
+func TestConfig_validateAddrs(t *testing.T) {
+
+	// Prepare some reference strings that will be checked in the
+	// test below.
+	outboundAddr := outboundIP().String()
+	hostname, err := os.Hostname()
+	if err != nil {
+		t.Fatal(err)
+	}
+	hostAddr, err := lookupAddr(context.Background(), net.DefaultResolver, hostname)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if strings.Contains(hostAddr, ":") {
+		hostAddr = "[" + hostAddr + "]"
+	}
+
+	tests := []struct {
+		expErr string
+		in     addrs
+		exp    addrs
+	}{
+		// Default values; addresses set empty.
+		{"",
+			addrs{"", ""},
+			addrs{":10101", ":10101"}},
+		{"",
+			addrs{":", ""},
+			addrs{":10101", ":10101"}},
+		{"",
+			addrs{"", ":"},
+			addrs{":10101", ":10101"}},
+		{"",
+			addrs{":", ":"},
+			addrs{":10101", ":10101"}},
+		// Listener :port.
+		{"",
+			addrs{":1234", ""},
+			addrs{":1234", ":1234"}},
+		// Listener with host:port.
+		{"",
+			addrs{hostAddr + ":10101", ""},
+			addrs{hostAddr + ":10101", hostAddr + ":10101"}},
+		// Listener with host:.
+		{"",
+			addrs{hostAddr + ":", ""},
+			addrs{hostAddr + ":10101", hostAddr + ":10101"}},
+		// Listener with scheme:.
+		{"",
+			addrs{"http://" + hostAddr + ":", ""},
+			addrs{"http://" + hostAddr + ":10101", "http://" + hostAddr + ":10101"}},
+		// Listener with localhost:port.
+		{"",
+			addrs{"localhost:1234", ""},
+			addrs{"localhost:1234", "localhost:1234"}},
+		// Listener with localhost:.
+		{"",
+			addrs{"localhost:", ""},
+			addrs{"localhost:10101", "localhost:10101"}},
+		// Listener and advertise addresses.
+		{"",
+			addrs{hostAddr + ":1234", hostAddr + ":"},
+			addrs{hostAddr + ":1234", hostAddr + ":1234"}},
+		// Explicit port number in advertise addr.
+		{"",
+			addrs{hostAddr + ":1234", hostAddr + ":7890"},
+			addrs{hostAddr + ":1234", hostAddr + ":7890"}},
+		// Use a non-numeric port number.
+		{"",
+			addrs{":postgresql", ""},
+			addrs{":5432", ":5432"}},
+		// Advertise port 0 means reuse listen port.
+		{"",
+			addrs{":1234", ":0"},
+			addrs{":1234", ":1234"}},
+		// Listen on all interfaces. Determine advertise address.
+		{"",
+			addrs{"0.0.0.0:1234", ""},
+			addrs{"0.0.0.0:1234", outboundAddr + ":1234"}},
+		// Expected errors.
+
+		// Missing port number.
+		{"missing port in address",
+			addrs{"localhost", ""},
+			addrs{}},
+		{"missing port in address",
+			addrs{":1234", "localhost"},
+			addrs{}},
+		// Invalid port number.
+		{"invalid port",
+			addrs{"localhost:-1234", ""},
+			addrs{}},
+		{"validating advertise address",
+			addrs{"localhost:foo", ""},
+			addrs{}},
+		{"no such host",
+			addrs{"333.333.333.333:1234", ""},
+			addrs{}},
+	}
+
+	for i, test := range tests {
+		c := NewConfig()
+
+		c.Bind = test.in.bind
+		c.Advertise = test.in.advertise
+
+		err := c.validateAddrs(context.Background())
+
+		if err != nil && test.expErr == "" {
+			t.Fatal(err)
+		} else if err == nil && test.expErr != "" {
+			t.Fatalf("test %d: expected error string to contain %s, but got no error", i, test.expErr)
+		} else if err != nil && test.expErr != "" {
+			if strings.Contains(err.Error(), test.expErr) {
+				continue
+			} else {
+				t.Fatalf("test %d: expected error string to contain %s, but got %s", i, test.expErr, err.Error())
+			}
+		}
+
+		if c.Bind != test.exp.bind {
+			t.Fatalf("test %d: bind address: expected %s, but got %s", i, test.exp.bind, c.Bind)
+		} else if c.Advertise != test.exp.advertise {
+			t.Fatalf("test %d: advertise address: expected %s, but got %s", i, test.exp.advertise, c.Advertise)
+		}
+	}
+}

server/server.go

@@ -21,6 +21,7 @@ package server
 
 import (
 	"bytes"
+	"context"
 	"crypto/tls"
 	"io"
 	"log"
@@ -78,6 +79,7 @@ type Command struct {
 	Handler      pilosa.Handler
 	API          *pilosa.API
 	ln           net.Listener
+	listenURI    *pilosa.URI
 	closeTimeout time.Duration
 
 	serverOptions []pilosa.ServerOption
@@ -151,7 +153,7 @@ func (m *Command) Start() (err error) {
 		return errors.Wrap(err, "opening server")
 	}
 
-	m.logger.Printf("listening as %s\n", m.API.Node().URI)
+	m.logger.Printf("listening as %s\n", m.listenURI)
 
 	return nil
 }
@@ -187,6 +189,15 @@ func (m *Command) SetupServer() error {
 	}
 	m.logger.Printf("%s %s, build time %s\n", productName, pilosa.Version, pilosa.BuildTime)
 
+	// validateAddrs sets the appropriate values for Bind and Advertise
+	// based on the inputs. It is not responsible for applying defaults, although
+	// it does provide a non-zero port (10101) in the case where no port is specified.
+	// The alternative would be to use port 0, which would choose a random port, but
+	// currently that's not what we want.
+	if err := m.Config.validateAddrs(context.Background()); err != nil {
+		return errors.Wrap(err, "validating addresses")
+	}
+
 	uri, err := pilosa.AddressWithDefaults(m.Config.Bind)
 	if err != nil {
 		return errors.Wrap(err, "processing bind address")
@@ -231,8 +242,20 @@ func (m *Command) SetupServer() error {
 		uri.SetPort(uint16(m.ln.Addr().(*net.TCPAddr).Port))
 	}
 
+	// Save listenURI for later reference.
+	m.listenURI = uri
+
 	c := http.GetHTTPClient(TLSConfig)
 
+	// Get advertise address as uri.
+	advertiseURI, err := pilosa.AddressWithDefaults(m.Config.Advertise)
+	if err != nil {
+		return errors.Wrap(err, "processing advertise address")
+	}
+	if advertiseURI.Port == 0 {
+		advertiseURI.SetPort(uri.Port)
+	}
+
 	// Primary store configuration is handled automatically now.
 	if m.Config.Translation.PrimaryURL != "" {
 		m.logger.Printf("DEPRECATED: The primary-url configuration option is no longer used.")
@@ -258,7 +281,7 @@ func (m *Command) SetupServer() error {
 		pilosa.OptServerSystemInfo(gopsutil.NewSystemInfo()),
 		pilosa.OptServerGCNotifier(gcnotify.NewActiveGCNotifier()),
 		pilosa.OptServerStatsClient(statsClient),
-		pilosa.OptServerURI(uri),
+		pilosa.OptServerURI(advertiseURI),
 		pilosa.OptServerInternalClient(http.NewInternalClientFromURI(uri, c)),
 		pilosa.OptServerPrimaryTranslateStoreFunc(http.NewTranslateStore),
 		pilosa.OptServerClusterDisabled(m.Config.Cluster.Disabled, m.Config.Cluster.Hosts),
@@ -295,7 +318,6 @@ func (m *Command) SetupServer() error {
 		http.OptHandlerCloseTimeout(m.closeTimeout),
 	)
 	return errors.Wrap(err, "new handler")
-
 }
 
 // setupNetworking sets up internode communication based on the configuration.
@@ -310,7 +332,7 @@ func (m *Command) setupNetworking() error {
 	}
 
 	// get the host portion of addr to use for binding
-	gossipHost := m.API.Node().URI.Host
+	gossipHost := m.listenURI.Host
 	m.gossipTransport, err = gossip.NewTransport(gossipHost, gossipPort, m.logger.Logger())
 	if err != nil {
 		return errors.Wrap(err, "getting transport")
@@ -320,6 +342,7 @@ func (m *Command) setupNetworking() error {
 		m.Config.Gossip,
 		m.API,
 		gossip.WithLogOutput(&filteredWriter{logOutput: m.logOutput, v: m.Config.Verbose}),
+		gossip.WithPilosaLogger(m.logger),
 		gossip.WithTransport(m.gossipTransport),
 	)
 	if err != nil {

server/server_test.go

@@ -293,7 +293,7 @@ func TestMain_GroupBy(t *testing.T) {
 	}
 
 	// Query row.
-	if res, err := m.QueryProtobuf("i", `GroupBy(Rows(field="generalk"), Rows(field="subk"))`); err != nil {
+	if res, err := m.QueryProtobuf("i", `GroupBy(Rows(generalk), Rows(subk))`); err != nil {
 		t.Fatal(err)
 	} else {
 		test.CheckGroupBy(t, expected, res.Results[0].([]pilosa.GroupCount))
@@ -468,7 +468,6 @@ func TestClusteringNodesReplica1(t *testing.T) {
 	// Create new main with the same config.
 	config := cluster[2].Command.Config
 	config.Translation.MapSize = 100000
-	// config.Bind = cluster[2].API.Node().URI.HostPort()
 
 	// this isn't necessary, but makes the test run way faster
 	config.Gossip.Port = strconv.Itoa(int(cluster[2].Command.GossipTransport().URI.Port))

server_internal_test.go

@@ -38,7 +38,7 @@ func TestCountOpenFiles(t *testing.T) {
 
 func TestMonitorAntiEntropyZero(t *testing.T) {
 
-	td, err := ioutil.TempDir("", "")
+	td, err := ioutil.TempDir(*TempDir, "")
 	if err != nil {
 		t.Fatalf("getting temp dir: %v", err)
 	}

translate.go

@@ -372,7 +372,6 @@ func (s *TranslateFile) monitorReplication() {
 		if err := s.replicate(ctx); err != nil {
 			s.logger.Printf("pilosa: replication error: %s", err)
 		}
-
 		select {
 		case <-ctx.Done():
 			return
@@ -412,22 +411,42 @@ func (s *TranslateFile) replicate(ctx context.Context) error {
 	// Wrap in bufferred I/O so it implements io.ByteReader.
 	bufr := bufio.NewReader(rc)
 
+	// we need a way to make an asynchronous routine hand us back an error,
+	// but we might not still be there to get it. so we have a buffer.
+	chErr := make(chan error, 1)
+
 	// Continually read new entries from primary and append to local store.
 	for {
 		// Read next available entry.
 		var entry LogEntry
-		if _, err := entry.ReadFrom(bufr); err == io.EOF {
+		if _, err = entry.ReadFrom(bufr); err == io.EOF {
 			return nil
 		} else if err != nil {
 			return err
 		}
-		s.mu.Lock()
-		// Write to local store.
-		if err := s.appendEntry(&entry); err != nil {
-			s.mu.Unlock()
-			return err
+		// note: we should never end up spawning two of this goroutine
+		// at once. either we end up reading the error from chErr below,
+		// and this loop continues, or we don't, and the whole function
+		// returns. if the function returns, we can write that single
+		// error to the empty channel with a buffer of 1, the goroutine
+		// terminates, and chErr becomes garbage-collectable.
+		go func() {
+			s.mu.Lock()
+			defer s.mu.Unlock()
+			// Write to local store.
+			err = s.appendEntry(&entry)
+			chErr <- err
+		}()
+		select {
+		case err = <-chErr:
+			if err != nil {
+				return err
+			}
+		case <-s.replicationClosing:
+			return nil
+		case <-ctx.Done():
+			return nil
 		}
-		s.mu.Unlock()
 	}
 }
 

translate_test.go

@@ -803,7 +803,7 @@ type TranslateFile struct {
 }
 
 func NewTranslateFile() *TranslateFile {
-	f, err := ioutil.TempFile("", "")
+	f, err := ioutil.TempFile(*TempDir, "")
 	if err != nil {
 		panic(err)
 	}

utils_internal_test.go

@@ -212,7 +212,7 @@ func (t *ClusterCluster) addCluster(i int, saveTopology bool) (*cluster, error)
 	t.common.Nodes = append(t.common.Nodes, node)
 
 	// create node-specific temp directory
-	path, err := ioutil.TempDir("", fmt.Sprintf("pilosa-cluster-node-%d-", i))
+	path, err := ioutil.TempDir(*TempDir, fmt.Sprintf("pilosa-cluster-node-%d-", i))
 	if err != nil {
 		return nil, err
 	}

view_internal_test.go

@@ -24,7 +24,7 @@ import (
 
 // mustOpenView returns a new instance of View with a temporary path.
 func mustOpenView(index, field, name string) *view {
-	path, err := ioutil.TempDir("", "pilosa-view-")
+	path, err := ioutil.TempDir(*TempDir, "pilosa-view-")
 	if err != nil {
 		panic(err)
 	}