From a242b8cbafcbd7029026474c893e82935335ab08 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 30 Jan 2019 16:01:39 -0600 Subject: [PATCH] add from/to range arguments to Rows() --- docs/query-language.md | 8 ++++- executor.go | 82 +++++++++++++++++++++++++++++++++++------- executor_test.go | 42 ++++++++++++++++++++-- 3 files changed, 115 insertions(+), 17 deletions(-) diff --git a/docs/query-language.md b/docs/query-language.md index 963b6c8e4..136697ff1 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -812,10 +812,12 @@ Options(Row(f1=10), shards=[0, 2]) {"attrs":{},"columns":[100, 2097152]} ``` +#### Rows + **Spec:** ``` -Rows(, previous=, limit=, column=) +Rows(, previous=, limit=, column=, from=, to=) ``` **Description:** @@ -832,6 +834,10 @@ is given, the number of rowIDs returned will be less than or equal to result sets. Results are always ordered, so setting `previous` as the last result of the previous request will start from the next available row. +If the field is of type `time`, the `from` and `to` arguments can be provided +to restrict the result to a specific time span. If `from` and `to` are +not provided, the field must be configured with `noStandardView` set to +`false`. **Result Type:** Object with `"rows" or "keys" and an array of integers or strings respectively.` diff --git a/executor.go b/executor.go index 20fc298c5..1fbb72ce8 100644 --- a/executor.go +++ b/executor.go @@ -1152,16 +1152,59 @@ func (e *executor) executeRowsShard(_ context.Context, index string, fieldName s return nil, ErrFieldNotFound } - // Rows query does not currently support a `time` field that has - // `noStandardView: true`. - // TODO https://github.com/pilosa/pilosa/issues/1783 - if f.Type() == FieldTypeTime && f.options.NoStandardView { - return nil, errors.New("Rows() query on time field with no standard view is not currently supported") - } + // rowIDs is the result set. + var rowIDs RowIDs - frag := e.Holder.fragment(index, fieldName, viewStandard, shard) - if frag == nil { - return make(RowIDs, 0), nil + // views contains the list of views to inspect (and merge) + // in order to represent `Rows` for the field. + var views []string = []string{viewStandard} + + // Handle `time` fields. + if f.Type() == FieldTypeTime { + var err error + + // Parse "from" time, if set. + var fromTime time.Time + if _, ok := c.Args["from"]; ok { + switch v := c.Args["from"].(type) { + case string: + if fromTime, err = time.Parse(TimeFormat, v); err != nil { + return nil, errors.New("cannot parse Row() 'from' time") + } + case int64: + fromTime = time.Unix(v, 0).UTC() + default: + return nil, errors.New("Row() 'from' arg must be a timestamp") + } + } + + // Parse "to" time, if set. + var toTime time.Time + if _, ok := c.Args["to"]; ok { + switch v := c.Args["to"].(type) { + case string: + if toTime, err = time.Parse(TimeFormat, v); err != nil { + return nil, errors.New("cannot parse Row() 'to' time") + } + case int64: + toTime = time.Unix(v, 0).UTC() + default: + return nil, errors.New("Row() 'to' arg must be a timestamp") + } + } + + if !fromTime.IsZero() && !toTime.IsZero() { + // If no quantum exists then return an empty result set. + q := f.TimeQuantum() + if q == "" { + return rowIDs, nil + } + + // Determine the views based on the specified time range. + views = viewsByTimeRange(viewStandard, fromTime, toTime, q) + } else if f.options.NoStandardView { + return nil, errors.New("Rows() query on time field with no standard view requires a date range") + } } start := uint64(0) @@ -1177,17 +1220,30 @@ func (e *executor) executeRowsShard(_ context.Context, index string, fieldName s } else if ok { colShard := columnID >> shardWidthExponent if colShard != shard { - return RowIDs{}, nil + return rowIDs, nil } filters = append(filters, filterColumn(columnID)) } - if limit, hasLimit, err := c.UintArg("limit"); err != nil { + + limit := int(^uint(0) >> 1) + if lim, hasLimit, err := c.UintArg("limit"); err != nil { return nil, errors.Wrap(err, "getting limit") } else if hasLimit { - filters = append(filters, filterWithLimit(limit)) + filters = append(filters, filterWithLimit(lim)) + limit = int(lim) } - return frag.rows(start, filters...), nil + for _, view := range views { + frag := e.Holder.fragment(index, fieldName, view, shard) + if frag == nil { + continue + } + + viewRows := frag.rows(start, filters...) + rowIDs = rowIDs.merge(viewRows, limit) + } + + return rowIDs, nil } func (e *executor) executeRowShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { diff --git a/executor_test.go b/executor_test.go index 146585d33..20622e270 100644 --- a/executor_test.go +++ b/executor_test.go @@ -3133,11 +3133,47 @@ func TestExecutor_Execute_Rows(t *testing.T) { func TestExecutor_Execute_RowsTime(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - c.CreateField(t, "i", pilosa.IndexOptions{}, "t", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), true)) + c.CreateField(t, "i", pilosa.IndexOptions{}, "f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), true)) - exp := "executing: Rows() query on time field with no standard view is not currently supported" - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Rows(field=t)`}); err == nil || err.Error() != exp { + writeQuery := fmt.Sprintf(` + Set(9, f=1, 2001-01-01T00:00) + Set(9, f=2, 2002-01-01T00:00) + Set(9, f=3, 2003-01-01T00:00) + Set(9, f=4, 2004-01-01T00:00) + + Set(%d, f=13, 2003-02-02T00:00) + `, pilosa.ShardWidth+9) + readQueries := []string{ + `Rows(field=f, from=1999-12-31T00:00, to=2002-01-01T03:00)`, + `Rows(field=f, from=2002-01-01T00:00, to=2004-01-01T00:00)`, + `Rows(field=f, from=1990-01-01T00:00, to=1999-01-01T00:00)`, + } + expResults := [][]uint64{ + {1}, + {2, 3, 13}, + {}, + } + + // Make sure that date range is enforced when there's no standard view + // or if only a partial date range is provided. + exp := "executing: Rows() query on time field with no standard view requires a date range" + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Rows(field=f)`}); err == nil || err.Error() != exp { t.Fatalf("expected error: %s", exp) + } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Rows(field=f, from=1999-12-31T00:00)`}); err == nil || err.Error() != exp { + t.Fatalf("expected error: %s", exp) + } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Rows(field=f, to=1999-12-31T00:00)`}); err == nil || err.Error() != exp { + t.Fatalf("expected error: %s", exp) + } + + responses := runCallTest(t, writeQuery, readQueries, + nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), true)) + + for i := range responses { + t.Run(fmt.Sprintf("response-%d", i), func(t *testing.T) { + if rows := responses[i].Results[0].(pilosa.RowIdentifiers).Rows; !reflect.DeepEqual(rows, expResults[i]) { + t.Fatalf("unexpected rows: %+v", rows) + } + }) } }