Merge branch 'master' into remove-arm64-binary

README.md

@@ -40,7 +40,7 @@ See our [Documentation](https://www.pilosa.com/docs/) for information about inst
     and verify that it's running:
     
     ```shell
-    curl localhost:10101/nodes
+    curl localhost:10101/status
     ```
 
 3.  Follow along with the [Sample Project](https://www.pilosa.com/docs/getting-started/#sample-project) to get a better understanding of Pilosa's capabilities.

docs/api-reference.md

@@ -376,6 +376,27 @@ curl -XGET localhost:10101/status
 }
 ```
 
+### Get active queries
+
+`GET /queries`
+
+Returns the set of active queries. Supports pretty printing in `text/plain` format or JSON output in `application/json` format.
+Also includes the amount of time that the query has been running (in nanoseconds when using JSON).
+
+```request
+curl -XGET localhost:10101/queries
+```
+```response
+182.412µs  All()
+```
+
+```request
+curl -XGET -H "Accept: application/json" localhost:10101/queries
+```
+```response
+[{"query":"All()","age":135123}]
+```
+
 ### Recalculate Caches
 
 `POST /recalculate-caches`

executor.go

@@ -4299,7 +4299,12 @@ func (e *executor) translateCall(ctx context.Context, indexName string, c *pql.C
 
 		fields := make([]*Field, len(c.Children))
 		for i, child := range c.Children {
-			fieldname := callArgString(child, "_field")
+			var fieldname string
+			if fieldname = callArgString(child, "_field"); fieldname == "" {
+				// TODO: it's unsettling that we expect "_field" but in some
+				// cases get "field". We should figure out why that happens.
+				fieldname = callArgString(child, "field")
+			}
 			field := idx.Field(fieldname)
 			if field == nil {
 				return errors.Wrapf(ErrFieldNotFound, "getting field '%s' from '%s'", fieldname, child)
@@ -4314,10 +4319,21 @@ func (e *executor) translateCall(ctx context.Context, indexName string, c *pql.C
 				if !ok {
 					return errors.New("prev value must be a string when field 'keys' option enabled")
 				}
-				// TODO: does this need to take field.ForeignIndex() into consideration?
-				id, err := e.Cluster.translateFieldKey(ctx, field, prevStr)
-				if err != nil {
-					return errors.Wrapf(err, "translating field key: %s", prevStr)
+				var id uint64
+				var err error
+				if fi := field.ForeignIndex(); fi != "" {
+					ids, err := e.Cluster.translateIndexKeys(ctx, fi, []string{prevStr})
+					if err != nil {
+						return errors.Wrap(err, "translating foreign index key in groupby previous")
+					}
+					if len(ids) == 1 {
+						id = ids[0]
+					}
+				} else {
+					id, err = e.Cluster.translateFieldKey(ctx, field, prevStr)
+					if err != nil {
+						return errors.Wrapf(err, "translating field key: %s", prevStr)
+					}
 				}
 				previous[i] = id
 			} else {
@@ -4350,7 +4366,7 @@ func (e *executor) translateResults(ctx context.Context, index string, idx *Inde
 	}
 
 	for i := range results {
-		results[i], err = e.translateResult(index, idx, calls[i], results[i], idMap)
+		results[i], err = e.translateResult(ctx, index, idx, calls[i], results[i], idMap)
 		if err != nil {
 			return err
 		}
@@ -4374,7 +4390,7 @@ func (e *executor) collectResultIDs(index string, idx *Index, call *pql.Call, re
 	return nil
 }
 
-func (e *executor) translateResult(index string, idx *Index, call *pql.Call, result interface{}, idSet map[uint64]string) (interface{}, error) {
+func (e *executor) translateResult(ctx context.Context, index string, idx *Index, call *pql.Call, result interface{}, idSet map[uint64]string) (interface{}, error) {
 	switch result := result.(type) {
 	case *Row:
 		if idx.Keys() {
@@ -4481,10 +4497,23 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res
 					return nil, ErrFieldNotFound
 				}
 				if field.Keys() {
-					// TODO: does this need to take field.ForeignIndex() into consideration?
-					key, err := field.TranslateStore().TranslateID(g.RowID)
-					if err != nil {
-						return nil, errors.Wrap(err, "translating row ID in Group")
+					var key string
+					var err error
+					if fi := field.ForeignIndex(); fi != "" && g.Value != nil {
+						val := uint64(*g.Value) // not worried about overflow here because it's a foreign key
+						keys, err := e.Cluster.translateIndexIDs(ctx, fi, []uint64{val})
+						if err != nil {
+							return nil, errors.Wrap(err, "translating foreign index in Group")
+						}
+						if len(keys) == 1 {
+							key = keys[0]
+							group[i].Value = nil // Remove value now that it has been translated.
+						}
+					} else {
+						key, err = field.TranslateStore().TranslateID(g.RowID)
+						if err != nil {
+							return nil, errors.Wrap(err, "translating row ID in Group")
+						}
 					}
 					group[i].RowKey = key
 				}

executor_test.go

@@ -4743,7 +4743,7 @@ func TestExecutor_Execute_Rows_Keys(t *testing.T) {
 			exp: []string{},
 		},
 		{
-			q: `Rows(f, like="__")`,
+			q:   `Rows(f, like="__")`,
 			exp: []string{"10", "11", "12", "13", "14", "15", "16", "17", "18"},
 		},
 	}
@@ -5189,6 +5189,43 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
 
 		})
 
+		// Foreign Index
+		c.CreateField(t, "fip", pilosa.IndexOptions{Keys: true}, "parent")
+		c.CreateField(t, "fic", pilosa.IndexOptions{}, "child",
+			pilosa.OptFieldTypeInt(0, math.MaxInt64),
+			pilosa.OptFieldForeignIndex("fip"),
+		)
+		// Set data on the parent so we have some index keys.
+		c.Query(t, "fip", `
+			Set("one", parent=1)
+			Set("two", parent=2)
+			Set("three", parent=3)
+			Set("four", parent=4)
+			Set("five", parent=5)
+		`)
+		// Set data on the child to align with the foreign index keys.
+		c.Query(t, "fic", `
+			Set(1, child="one")
+			Set(2, child="one")
+			Set(3, child="one")
+			Set(4, child="three")
+			Set(5, child="three")
+			Set(6, child="five")
+		`)
+
+		t.Run("test foreign index with keys", func(t *testing.T) {
+			// the execututor returns row IDs when the field has keys, so they should be included in the target.
+			// because the order is determined by the partitioned index key, they seem out of order.
+			expected := []pilosa.GroupCount{
+				{Group: []pilosa.FieldRow{{Field: "child", RowID: 0, RowKey: "one"}}, Count: 3},
+				{Group: []pilosa.FieldRow{{Field: "child", RowID: 1, RowKey: "five"}}, Count: 1},
+				{Group: []pilosa.FieldRow{{Field: "child", RowID: 2, RowKey: "three"}}, Count: 2},
+			}
+
+			results := c.Query(t, "fic", `GroupBy(Rows(child))`).Results[0].([]pilosa.GroupCount)
+			test.CheckGroupBy(t, expected, results)
+		})
+
 	}
 	for size := range []int{1, 3} {
 		t.Run(fmt.Sprintf("%d_nodes", size), func(t *testing.T) {

pilosa.go

@@ -53,8 +53,7 @@ var (
 	ErrInvalidView      = errors.New("invalid view")
 	ErrInvalidCacheType = errors.New("invalid cache type")
 
-	ErrName  = errors.New("invalid index or field name, must match [a-z][a-z0-9_-]* and contain at most 230 characters")
-	ErrLabel = errors.New("invalid row or column label, must match [A-Za-z0-9_-]")
+	ErrName = errors.New("invalid index or field name, must match [a-z][a-z0-9_-]* and contain at most 230 characters")
 
 	// ErrFragmentNotFound is returned when a fragment does not exist.
 	ErrFragmentNotFound = errors.New("fragment not found")

transaction.go

@@ -17,6 +17,7 @@ package pilosa
 import (
 	"context"
 	"encoding/json"
+	"regexp"
 	"sync"
 	"time"
 
@@ -24,6 +25,8 @@ import (
 	"github.com/pkg/errors"
 )
 
+var txIDRegexp = regexp.MustCompile("^[A-Za-z0-9_-]*$")
+
 // Transaction contains information related to a block of work that
 // needs to be tracked and spans multiple API calls.
 type Transaction struct {
@@ -93,6 +96,10 @@ func (tm *TransactionManager) Start(ctx context.Context, id string, timeout time
 	tm.mu.Lock()
 	defer tm.mu.Unlock()
 
+	if !txIDRegexp.Match([]byte(id)) {
+		return nil, errors.New("invalid transaction ID, must match [A-Za-z0-9_-]")
+	}
+
 	trnsMap, err := tm.store.List()
 	if err != nil {
 		return nil, errors.Wrap(err, "listing transactions in Start")