From aa0cfecdb33df6a6bcd196fc40b5bc83ca29ce28 Mon Sep 17 00:00:00 2001 From: alisharawal <65546978+alisharawal@users.noreply.github.com> Date: Wed, 20 May 2020 11:37:22 -0500 Subject: [PATCH 1/6] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8ccae2b91..3d7af147c 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ See our [Documentation](https://www.pilosa.com/docs/) for information about inst and verify that it's running: ```shell - curl localhost:10101/nodes + curl localhost:10101/status ``` 3. Follow along with the [Sample Project](https://www.pilosa.com/docs/getting-started/#sample-project) to get a better understanding of Pilosa's capabilities. From 7c37ececfd3642f48345303b54e3cae7d5458fa1 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Fri, 10 Jul 2020 17:58:47 -0500 Subject: [PATCH 2/6] Restrict allowed characters in transaction IDs --- pilosa.go | 3 +-- transaction.go | 7 +++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/pilosa.go b/pilosa.go index 481c8ff07..f7eb33f43 100644 --- a/pilosa.go +++ b/pilosa.go @@ -53,8 +53,7 @@ var ( ErrInvalidView = errors.New("invalid view") ErrInvalidCacheType = errors.New("invalid cache type") - ErrName = errors.New("invalid index or field name, must match [a-z][a-z0-9_-]* and contain at most 230 characters") - ErrLabel = errors.New("invalid row or column label, must match [A-Za-z0-9_-]") + ErrName = errors.New("invalid index or field name, must match [a-z][a-z0-9_-]* and contain at most 230 characters") // ErrFragmentNotFound is returned when a fragment does not exist. ErrFragmentNotFound = errors.New("fragment not found") diff --git a/transaction.go b/transaction.go index ba7063907..1923bf91b 100644 --- a/transaction.go +++ b/transaction.go @@ -17,6 +17,7 @@ package pilosa import ( "context" "encoding/json" + "regexp" "sync" "time" @@ -24,6 +25,8 @@ import ( "github.com/pkg/errors" ) +var txIDRegexp = regexp.MustCompile("^[A-Za-z0-9_-]$") + // Transaction contains information related to a block of work that // needs to be tracked and spans multiple API calls. type Transaction struct { @@ -93,6 +96,10 @@ func (tm *TransactionManager) Start(ctx context.Context, id string, timeout time tm.mu.Lock() defer tm.mu.Unlock() + if !txIDRegexp.Match([]byte(id)) { + return nil, errors.New("invalid transaction ID, must match [A-Za-z0-9_-]") + } + trnsMap, err := tm.store.List() if err != nil { return nil, errors.Wrap(err, "listing transactions in Start") From 65febc37fd9d0db2003811b39720347bb936ee84 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Fri, 10 Jul 2020 18:07:17 -0500 Subject: [PATCH 3/6] Allow '' in ID regex --- transaction.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transaction.go b/transaction.go index 1923bf91b..2af2c76df 100644 --- a/transaction.go +++ b/transaction.go @@ -25,7 +25,7 @@ import ( "github.com/pkg/errors" ) -var txIDRegexp = regexp.MustCompile("^[A-Za-z0-9_-]$") +var txIDRegexp = regexp.MustCompile("^[A-Za-z0-9_-]*$") // Transaction contains information related to a block of work that // needs to be tracked and spans multiple API calls. From 8f6b876af0af947c45ffacb6ce5ad75a45efe00a Mon Sep 17 00:00:00 2001 From: Travis Date: Fri, 10 Jul 2020 22:41:55 -0500 Subject: [PATCH 4/6] get ForeignIndex keys in GroupBy --- executor.go | 25 +++++++++++++++++++------ executor_test.go | 39 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 57 insertions(+), 7 deletions(-) diff --git a/executor.go b/executor.go index 42abe099c..3e50c3c95 100644 --- a/executor.go +++ b/executor.go @@ -4350,7 +4350,7 @@ func (e *executor) translateResults(ctx context.Context, index string, idx *Inde } for i := range results { - results[i], err = e.translateResult(index, idx, calls[i], results[i], idMap) + results[i], err = e.translateResult(ctx, index, idx, calls[i], results[i], idMap) if err != nil { return err } @@ -4374,7 +4374,7 @@ func (e *executor) collectResultIDs(index string, idx *Index, call *pql.Call, re return nil } -func (e *executor) translateResult(index string, idx *Index, call *pql.Call, result interface{}, idSet map[uint64]string) (interface{}, error) { +func (e *executor) translateResult(ctx context.Context, index string, idx *Index, call *pql.Call, result interface{}, idSet map[uint64]string) (interface{}, error) { switch result := result.(type) { case *Row: if idx.Keys() { @@ -4481,10 +4481,23 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res return nil, ErrFieldNotFound } if field.Keys() { - // TODO: does this need to take field.ForeignIndex() into consideration? - key, err := field.TranslateStore().TranslateID(g.RowID) - if err != nil { - return nil, errors.Wrap(err, "translating row ID in Group") + var key string + var err error + if fi := field.ForeignIndex(); fi != "" && g.Value != nil { + val := uint64(*g.Value) // not worried about overflow here because it's a foreign key + keys, err := e.Cluster.translateIndexIDs(ctx, fi, []uint64{val}) + if err != nil { + return nil, errors.Wrap(err, "translating foreign index in Group") + } + if len(keys) == 1 { + key = keys[0] + group[i].Value = nil // Remove value now that it has been translated. + } + } else { + key, err = field.TranslateStore().TranslateID(g.RowID) + if err != nil { + return nil, errors.Wrap(err, "translating row ID in Group") + } } group[i].RowKey = key } diff --git a/executor_test.go b/executor_test.go index 4a39a7713..32fc01b27 100644 --- a/executor_test.go +++ b/executor_test.go @@ -4743,7 +4743,7 @@ func TestExecutor_Execute_Rows_Keys(t *testing.T) { exp: []string{}, }, { - q: `Rows(f, like="__")`, + q: `Rows(f, like="__")`, exp: []string{"10", "11", "12", "13", "14", "15", "16", "17", "18"}, }, } @@ -5189,6 +5189,43 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { }) + // Foreign Index + c.CreateField(t, "fip", pilosa.IndexOptions{Keys: true}, "parent") + c.CreateField(t, "fic", pilosa.IndexOptions{}, "child", + pilosa.OptFieldTypeInt(0, math.MaxInt64), + pilosa.OptFieldForeignIndex("fip"), + ) + // Set data on the parent so we have some index keys. + c.Query(t, "fip", ` + Set("one", parent=1) + Set("two", parent=2) + Set("three", parent=3) + Set("four", parent=4) + Set("five", parent=5) + `) + // Set data on the child to align with the foreign index keys. + c.Query(t, "fic", ` + Set(1, child="one") + Set(2, child="one") + Set(3, child="one") + Set(4, child="three") + Set(5, child="three") + Set(6, child="five") + `) + + t.Run("test foreign index with keys", func(t *testing.T) { + // the execututor returns row IDs when the field has keys, so they should be included in the target. + // because the order is determined by the partitioned index key, they seem out of order. + expected := []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{{Field: "child", RowID: 0, RowKey: "one"}}, Count: 3}, + {Group: []pilosa.FieldRow{{Field: "child", RowID: 1, RowKey: "five"}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "child", RowID: 2, RowKey: "three"}}, Count: 2}, + } + + results := c.Query(t, "fic", `GroupBy(Rows(child))`).Results[0].([]pilosa.GroupCount) + test.CheckGroupBy(t, expected, results) + }) + } for size := range []int{1, 3} { t.Run(fmt.Sprintf("%d_nodes", size), func(t *testing.T) { From a9dc8b8add01f7afc20d6d4e69f2a0436bd09d7a Mon Sep 17 00:00:00 2001 From: Travis Date: Sat, 11 Jul 2020 10:18:45 -0500 Subject: [PATCH 5/6] translate GroupBy previous value from foreign index --- executor.go | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/executor.go b/executor.go index 3e50c3c95..3812c8451 100644 --- a/executor.go +++ b/executor.go @@ -4299,7 +4299,12 @@ func (e *executor) translateCall(ctx context.Context, indexName string, c *pql.C fields := make([]*Field, len(c.Children)) for i, child := range c.Children { - fieldname := callArgString(child, "_field") + var fieldname string + if fieldname = callArgString(child, "_field"); fieldname == "" { + // TODO: it's unsettling that we expect "_field" but in some + // cases get "field". We should figure out why that happens. + fieldname = callArgString(child, "field") + } field := idx.Field(fieldname) if field == nil { return errors.Wrapf(ErrFieldNotFound, "getting field '%s' from '%s'", fieldname, child) @@ -4314,10 +4319,21 @@ func (e *executor) translateCall(ctx context.Context, indexName string, c *pql.C if !ok { return errors.New("prev value must be a string when field 'keys' option enabled") } - // TODO: does this need to take field.ForeignIndex() into consideration? - id, err := e.Cluster.translateFieldKey(ctx, field, prevStr) - if err != nil { - return errors.Wrapf(err, "translating field key: %s", prevStr) + var id uint64 + var err error + if fi := field.ForeignIndex(); fi != "" { + ids, err := e.Cluster.translateIndexKeys(ctx, fi, []string{prevStr}) + if err != nil { + return errors.Wrap(err, "translating foreign index key in groupby previous") + } + if len(ids) == 1 { + id = ids[0] + } + } else { + id, err = e.Cluster.translateFieldKey(ctx, field, prevStr) + if err != nil { + return errors.Wrapf(err, "translating field key: %s", prevStr) + } } previous[i] = id } else { From b442ff83361782c210f0c9a23b47d2e4bdc89ec6 Mon Sep 17 00:00:00 2001 From: Jaden Weiss Date: Tue, 14 Jul 2020 15:58:59 -0400 Subject: [PATCH 6/6] add /queries endpoint to API reference --- docs/api-reference.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/api-reference.md b/docs/api-reference.md index e8a2ac1a3..c414d9bb5 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -376,6 +376,27 @@ curl -XGET localhost:10101/status } ``` +### Get active queries + +`GET /queries` + +Returns the set of active queries. Supports pretty printing in `text/plain` format or JSON output in `application/json` format. +Also includes the amount of time that the query has been running (in nanoseconds when using JSON). + +```request +curl -XGET localhost:10101/queries +``` +```response +182.412µs All() +``` + +```request +curl -XGET -H "Accept: application/json" localhost:10101/queries +``` +```response +[{"query":"All()","age":135123}] +``` + ### Recalculate Caches `POST /recalculate-caches`