diff --git a/README.md b/README.md index 1619c2b57..8057a469d 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ See our [Documentation](https://www.pilosa.com/docs/) for information about inst and verify that it's running: ```shell - curl localhost:10101/nodes + curl localhost:10101/status ``` 3. Follow along with the [Sample Project](https://www.pilosa.com/docs/getting-started/#sample-project) to get a better understanding of Pilosa's capabilities. diff --git a/docs/api-reference.md b/docs/api-reference.md index e8a2ac1a3..c414d9bb5 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -376,6 +376,27 @@ curl -XGET localhost:10101/status } ``` +### Get active queries + +`GET /queries` + +Returns the set of active queries. Supports pretty printing in `text/plain` format or JSON output in `application/json` format. +Also includes the amount of time that the query has been running (in nanoseconds when using JSON). + +```request +curl -XGET localhost:10101/queries +``` +```response +182.412µs All() +``` + +```request +curl -XGET -H "Accept: application/json" localhost:10101/queries +``` +```response +[{"query":"All()","age":135123}] +``` + ### Recalculate Caches `POST /recalculate-caches` diff --git a/executor.go b/executor.go index 42abe099c..3812c8451 100644 --- a/executor.go +++ b/executor.go @@ -4299,7 +4299,12 @@ func (e *executor) translateCall(ctx context.Context, indexName string, c *pql.C fields := make([]*Field, len(c.Children)) for i, child := range c.Children { - fieldname := callArgString(child, "_field") + var fieldname string + if fieldname = callArgString(child, "_field"); fieldname == "" { + // TODO: it's unsettling that we expect "_field" but in some + // cases get "field". We should figure out why that happens. + fieldname = callArgString(child, "field") + } field := idx.Field(fieldname) if field == nil { return errors.Wrapf(ErrFieldNotFound, "getting field '%s' from '%s'", fieldname, child) @@ -4314,10 +4319,21 @@ func (e *executor) translateCall(ctx context.Context, indexName string, c *pql.C if !ok { return errors.New("prev value must be a string when field 'keys' option enabled") } - // TODO: does this need to take field.ForeignIndex() into consideration? - id, err := e.Cluster.translateFieldKey(ctx, field, prevStr) - if err != nil { - return errors.Wrapf(err, "translating field key: %s", prevStr) + var id uint64 + var err error + if fi := field.ForeignIndex(); fi != "" { + ids, err := e.Cluster.translateIndexKeys(ctx, fi, []string{prevStr}) + if err != nil { + return errors.Wrap(err, "translating foreign index key in groupby previous") + } + if len(ids) == 1 { + id = ids[0] + } + } else { + id, err = e.Cluster.translateFieldKey(ctx, field, prevStr) + if err != nil { + return errors.Wrapf(err, "translating field key: %s", prevStr) + } } previous[i] = id } else { @@ -4350,7 +4366,7 @@ func (e *executor) translateResults(ctx context.Context, index string, idx *Inde } for i := range results { - results[i], err = e.translateResult(index, idx, calls[i], results[i], idMap) + results[i], err = e.translateResult(ctx, index, idx, calls[i], results[i], idMap) if err != nil { return err } @@ -4374,7 +4390,7 @@ func (e *executor) collectResultIDs(index string, idx *Index, call *pql.Call, re return nil } -func (e *executor) translateResult(index string, idx *Index, call *pql.Call, result interface{}, idSet map[uint64]string) (interface{}, error) { +func (e *executor) translateResult(ctx context.Context, index string, idx *Index, call *pql.Call, result interface{}, idSet map[uint64]string) (interface{}, error) { switch result := result.(type) { case *Row: if idx.Keys() { @@ -4481,10 +4497,23 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res return nil, ErrFieldNotFound } if field.Keys() { - // TODO: does this need to take field.ForeignIndex() into consideration? - key, err := field.TranslateStore().TranslateID(g.RowID) - if err != nil { - return nil, errors.Wrap(err, "translating row ID in Group") + var key string + var err error + if fi := field.ForeignIndex(); fi != "" && g.Value != nil { + val := uint64(*g.Value) // not worried about overflow here because it's a foreign key + keys, err := e.Cluster.translateIndexIDs(ctx, fi, []uint64{val}) + if err != nil { + return nil, errors.Wrap(err, "translating foreign index in Group") + } + if len(keys) == 1 { + key = keys[0] + group[i].Value = nil // Remove value now that it has been translated. + } + } else { + key, err = field.TranslateStore().TranslateID(g.RowID) + if err != nil { + return nil, errors.Wrap(err, "translating row ID in Group") + } } group[i].RowKey = key } diff --git a/executor_test.go b/executor_test.go index 4a39a7713..32fc01b27 100644 --- a/executor_test.go +++ b/executor_test.go @@ -4743,7 +4743,7 @@ func TestExecutor_Execute_Rows_Keys(t *testing.T) { exp: []string{}, }, { - q: `Rows(f, like="__")`, + q: `Rows(f, like="__")`, exp: []string{"10", "11", "12", "13", "14", "15", "16", "17", "18"}, }, } @@ -5189,6 +5189,43 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { }) + // Foreign Index + c.CreateField(t, "fip", pilosa.IndexOptions{Keys: true}, "parent") + c.CreateField(t, "fic", pilosa.IndexOptions{}, "child", + pilosa.OptFieldTypeInt(0, math.MaxInt64), + pilosa.OptFieldForeignIndex("fip"), + ) + // Set data on the parent so we have some index keys. + c.Query(t, "fip", ` + Set("one", parent=1) + Set("two", parent=2) + Set("three", parent=3) + Set("four", parent=4) + Set("five", parent=5) + `) + // Set data on the child to align with the foreign index keys. + c.Query(t, "fic", ` + Set(1, child="one") + Set(2, child="one") + Set(3, child="one") + Set(4, child="three") + Set(5, child="three") + Set(6, child="five") + `) + + t.Run("test foreign index with keys", func(t *testing.T) { + // the execututor returns row IDs when the field has keys, so they should be included in the target. + // because the order is determined by the partitioned index key, they seem out of order. + expected := []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{{Field: "child", RowID: 0, RowKey: "one"}}, Count: 3}, + {Group: []pilosa.FieldRow{{Field: "child", RowID: 1, RowKey: "five"}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "child", RowID: 2, RowKey: "three"}}, Count: 2}, + } + + results := c.Query(t, "fic", `GroupBy(Rows(child))`).Results[0].([]pilosa.GroupCount) + test.CheckGroupBy(t, expected, results) + }) + } for size := range []int{1, 3} { t.Run(fmt.Sprintf("%d_nodes", size), func(t *testing.T) { diff --git a/pilosa.go b/pilosa.go index 481c8ff07..f7eb33f43 100644 --- a/pilosa.go +++ b/pilosa.go @@ -53,8 +53,7 @@ var ( ErrInvalidView = errors.New("invalid view") ErrInvalidCacheType = errors.New("invalid cache type") - ErrName = errors.New("invalid index or field name, must match [a-z][a-z0-9_-]* and contain at most 230 characters") - ErrLabel = errors.New("invalid row or column label, must match [A-Za-z0-9_-]") + ErrName = errors.New("invalid index or field name, must match [a-z][a-z0-9_-]* and contain at most 230 characters") // ErrFragmentNotFound is returned when a fragment does not exist. ErrFragmentNotFound = errors.New("fragment not found") diff --git a/transaction.go b/transaction.go index ba7063907..2af2c76df 100644 --- a/transaction.go +++ b/transaction.go @@ -17,6 +17,7 @@ package pilosa import ( "context" "encoding/json" + "regexp" "sync" "time" @@ -24,6 +25,8 @@ import ( "github.com/pkg/errors" ) +var txIDRegexp = regexp.MustCompile("^[A-Za-z0-9_-]*$") + // Transaction contains information related to a block of work that // needs to be tracked and spans multiple API calls. type Transaction struct { @@ -93,6 +96,10 @@ func (tm *TransactionManager) Start(ctx context.Context, id string, timeout time tm.mu.Lock() defer tm.mu.Unlock() + if !txIDRegexp.Match([]byte(id)) { + return nil, errors.New("invalid transaction ID, must match [A-Za-z0-9_-]") + } + trnsMap, err := tm.store.List() if err != nil { return nil, errors.Wrap(err, "listing transactions in Start")