diff --git a/docs/glossary.md b/docs/glossary.md
index 9ff232454..173d37abc 100644
--- a/docs/glossary.md
+++ b/docs/glossary.md
@@ -52,9 +52,9 @@ nav = []
[Row](../data-model/#row): Rows are the fundamental vertical data axis within Pilosa. They are namespaced to each [field](#field) within an [index](#index). Represented as a [Bitmap](#bitmap).
-[Row (Ranged)](../query-language/#range-queries):: A [PQL](#pql) query that returns bits based on comparison to timestamps, set according to the [time quantum](#time-quantum).
+[Row (Ranged)](../query-language/#row-range):: A [PQL](#pql) query that returns bits based on comparison to timestamps, set according to the [time quantum](#time-quantum).
-[Row (BSI)](../query-language/#range-bsi):: A [PQL](#pql) query that returns bits based on comparison to integers stored in [BSI](#bsi) [fields](#field).
+[Row (BSI)](../query-language/#row-bsi):: A [PQL](#pql) query that returns bits based on comparison to integers stored in [BSI](#bsi) [fields](#field).
[Slice](../data-model/#shard): Prior to Pilosa 1.0, shards were known as slices.
diff --git a/docs/query-language.md b/docs/query-language.md
index 4dd838727..408a967bc 100644
--- a/docs/query-language.md
+++ b/docs/query-language.md
@@ -280,7 +280,7 @@ Store(, =)
**Description:**
-`Store` writes the results of to the specified row. If the row already exists, it will be replaced. The destination field must be of field type `set`.
+`Store` writes the results of `` to the specified row. If the row already exists, it will be replaced. The destination field must be of field type `set`.
**Result Type:** boolean
@@ -406,7 +406,7 @@ BSI range queries support the following operators:
`==` | equal-to, EQ | integer
`!=` | not-equal-to, NEQ | integer or `null`
-`<`, and `<=` can be chained together to represent a bounded interval. For example:
+A bounded interval can be specified by chaining the `<` and `<=` operators (but not others). For example:
```request
Row(50 < commitactivity < 150)
@@ -786,6 +786,7 @@ Options(, columnAttrs=, excludeColumns=, excludeRowAttrs=, previous=, limit=, column=)
+Rows(, previous=, limit=, column=, from=, to=)
```
**Description:**
@@ -831,6 +834,9 @@ is given, the number of rowIDs returned will be less than or equal to
result sets. Results are always ordered, so setting `previous` as the last
result of the previous request will start from the next available row.
+If the field is of type `time`, the `from` and `to` arguments can be provided
+to restrict the result to a specific time span. If `from` and `to` are
+not provided, the full range of existing data will be queried.
**Result Type:** Object with `"rows" or "keys" and an array of integers or strings respectively.`
@@ -852,6 +858,7 @@ Rows(blahk)
{"rows":null,"keys":["haha","zaaa","traa"]}
```
+#### Group By
**Spec:**
diff --git a/docs/tutorials.md b/docs/tutorials.md
index 01e525362..60f29b74d 100644
--- a/docs/tutorials.md
+++ b/docs/tutorials.md
@@ -544,7 +544,7 @@ curl localhost:10101/index/patients/query \
{"results":[{"attrs":{},"columns":[2,6,9]}]}
```
-You can find a list of supported range operators in the [Row (BSI) Query](../query-language/#range-bsi) documentation.
+You can find a list of supported range operators in the [Row (BSI) Query](../query-language/#row-bsi) documentation.
To find the average age of all patients, run a `Sum` query:
``` request
diff --git a/executor.go b/executor.go
index 20fc298c5..b345d434b 100644
--- a/executor.go
+++ b/executor.go
@@ -1152,16 +1152,75 @@ func (e *executor) executeRowsShard(_ context.Context, index string, fieldName s
return nil, ErrFieldNotFound
}
- // Rows query does not currently support a `time` field that has
- // `noStandardView: true`.
- // TODO https://github.com/pilosa/pilosa/issues/1783
- if f.Type() == FieldTypeTime && f.options.NoStandardView {
- return nil, errors.New("Rows() query on time field with no standard view is not currently supported")
- }
+ // rowIDs is the result set.
+ var rowIDs RowIDs
- frag := e.Holder.fragment(index, fieldName, viewStandard, shard)
- if frag == nil {
- return make(RowIDs, 0), nil
+ // views contains the list of views to inspect (and merge)
+ // in order to represent `Rows` for the field.
+ var views []string = []string{viewStandard}
+
+ // Handle `time` fields.
+ if f.Type() == FieldTypeTime {
+ var err error
+
+ // Parse "from" time, if set.
+ var fromTime time.Time
+ if v, ok := c.Args["from"]; ok {
+ if fromTime, err = parseTime(v); err != nil {
+ return nil, errors.Wrap(err, "parsing from time")
+ }
+ }
+
+ // Parse "to" time, if set.
+ var toTime time.Time
+ if v, ok := c.Args["to"]; ok {
+ if toTime, err = parseTime(v); err != nil {
+ return nil, errors.Wrap(err, "parsing to time")
+ }
+ }
+
+ // Calculate the views for a range as long as some piece of the range
+ // (from/to) are specified, or if there's no standard view to represent
+ // all dates.
+ if !fromTime.IsZero() || !toTime.IsZero() || f.options.NoStandardView {
+ // If no quantum exists then return an empty result set.
+ q := f.TimeQuantum()
+ if q == "" {
+ return rowIDs, nil
+ }
+
+ // Get min/max based on existing views.
+ var vs []string
+ for _, v := range f.views() {
+ vs = append(vs, v.name)
+ }
+ min, max := minMaxViews(vs, q)
+
+ // If min/max are empty, there were no time views.
+ if min == "" || max == "" {
+ return rowIDs, nil
+ }
+
+ // Convert min/max from string to time.Time.
+ minTime, err := timeOfView(min, false)
+ if err != nil {
+ return rowIDs, errors.Wrapf(err, "getting min time from view: %s", min)
+ }
+ if fromTime.IsZero() || fromTime.Before(minTime) {
+ fromTime = minTime
+ }
+
+ maxTime, err := timeOfView(max, true)
+ if err != nil {
+ return rowIDs, errors.Wrapf(err, "getting max time from view: %s", max)
+ }
+ if toTime.IsZero() || toTime.After(maxTime) {
+ toTime = maxTime
+ }
+
+ // Determine the views based on the specified time range.
+ views = viewsByTimeRange(viewStandard, fromTime, toTime, q)
+ }
}
start := uint64(0)
@@ -1177,17 +1236,30 @@ func (e *executor) executeRowsShard(_ context.Context, index string, fieldName s
} else if ok {
colShard := columnID >> shardWidthExponent
if colShard != shard {
- return RowIDs{}, nil
+ return rowIDs, nil
}
filters = append(filters, filterColumn(columnID))
}
- if limit, hasLimit, err := c.UintArg("limit"); err != nil {
+
+ limit := int(^uint(0) >> 1)
+ if lim, hasLimit, err := c.UintArg("limit"); err != nil {
return nil, errors.Wrap(err, "getting limit")
} else if hasLimit {
- filters = append(filters, filterWithLimit(limit))
+ filters = append(filters, filterWithLimit(lim))
+ limit = int(lim)
}
- return frag.rows(start, filters...), nil
+ for _, view := range views {
+ frag := e.Holder.fragment(index, fieldName, view, shard)
+ if frag == nil {
+ continue
+ }
+
+ viewRows := frag.rows(start, filters...)
+ rowIDs = rowIDs.merge(viewRows, limit)
+ }
+
+ return rowIDs, nil
}
func (e *executor) executeRowShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
@@ -1228,31 +1300,17 @@ func (e *executor) executeRowShard(ctx context.Context, index string, c *pql.Cal
// Parse "from" time, if set.
var fromTime time.Time
- if _, ok := c.Args["from"]; ok {
- switch v := c.Args["from"].(type) {
- case string:
- if fromTime, err = time.Parse(TimeFormat, v); err != nil {
- return nil, errors.New("cannot parse Row() 'from' time")
- }
- case int64:
- fromTime = time.Unix(v, 0).UTC()
- default:
- return nil, errors.New("Row() 'from' arg must be a timestamp")
+ if v, ok := c.Args["from"]; ok {
+ if fromTime, err = parseTime(v); err != nil {
+ return nil, errors.Wrap(err, "parsing from time")
}
}
// Parse "to" time, if set.
var toTime time.Time
- if _, ok := c.Args["to"]; ok {
- switch v := c.Args["to"].(type) {
- case string:
- if toTime, err = time.Parse(TimeFormat, v); err != nil {
- return nil, errors.New("cannot parse Row() 'to' time")
- }
- case int64:
- toTime = time.Unix(v, 0).UTC()
- default:
- return nil, errors.New("Row() 'to' arg must be a timestamp")
+ if v, ok := c.Args["to"]; ok {
+ if toTime, err = parseTime(v); err != nil {
+ return nil, errors.Wrap(err, "parsing to time")
}
}
diff --git a/executor_test.go b/executor_test.go
index 146585d33..a36ddcf72 100644
--- a/executor_test.go
+++ b/executor_test.go
@@ -3131,13 +3131,51 @@ func TestExecutor_Execute_Rows(t *testing.T) {
}
func TestExecutor_Execute_RowsTime(t *testing.T) {
+ writeQuery := fmt.Sprintf(`
+ Set(9, f=1, 2001-01-01T00:00)
+ Set(9, f=2, 2002-01-01T00:00)
+ Set(9, f=3, 2003-01-01T00:00)
+ Set(9, f=4, 2004-01-01T00:00)
+
+ Set(%d, f=13, 2003-02-02T00:00)
+ `, pilosa.ShardWidth+9)
+ readQueries := []string{
+ `Rows(f, from=1999-12-31T00:00, to=2002-01-01T03:00)`,
+ `Rows(f, from=2002-01-01T00:00, to=2004-01-01T00:00)`,
+ `Rows(f, from=1990-01-01T00:00, to=1999-01-01T00:00)`,
+ `Rows(f)`,
+ `Rows(f, from=2002-01-01T00:00)`,
+ `Rows(f, to=2003-02-03T00:00)`,
+ }
+ expResults := [][]uint64{
+ {1},
+ {2, 3, 13},
+ {},
+ {1, 2, 3, 4, 13},
+ {2, 3, 4, 13},
+ {1, 2, 3, 13},
+ }
+
+ responses := runCallTest(t, writeQuery, readQueries,
+ nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), true))
+
+ for i := range responses {
+ t.Run(fmt.Sprintf("response-%d", i), func(t *testing.T) {
+ if rows := responses[i].Results[0].(pilosa.RowIdentifiers).Rows; !reflect.DeepEqual(rows, expResults[i]) {
+ t.Fatalf("unexpected rows: %+v", rows)
+ }
+ })
+ }
+}
+
+// Ensure that an empty time field returns empty Rows().
+func TestExecutor_Execute_RowsTimeEmpty(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
- c.CreateField(t, "i", pilosa.IndexOptions{}, "t", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), true))
-
- exp := "executing: Rows() query on time field with no standard view is not currently supported"
- if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Rows(field=t)`}); err == nil || err.Error() != exp {
- t.Fatalf("expected error: %s", exp)
+ c.CreateField(t, "i", pilosa.IndexOptions{}, "x", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), true))
+ rows := c.Query(t, "i", `Rows(x, from=1999-12-31T00:00, to=2002-01-01T03:00)`).Results[0].(pilosa.RowIdentifiers).Rows
+ if !reflect.DeepEqual(rows, []uint64{}) {
+ t.Fatalf("unexpected rows: %+v", rows)
}
}
diff --git a/time.go b/time.go
index 046b0223e..75ca543ea 100644
--- a/time.go
+++ b/time.go
@@ -17,6 +17,7 @@ package pilosa
import (
"errors"
"fmt"
+ "sort"
"strings"
"time"
)
@@ -214,3 +215,120 @@ func nextDayGTE(t time.Time, end time.Time) bool {
}
return end.After(next)
}
+
+// parseTime parses a string or int64 into a time.Time value.
+func parseTime(t interface{}) (time.Time, error) {
+ var err error
+ var calcTime time.Time
+ switch v := t.(type) {
+ case string:
+ if calcTime, err = time.Parse(TimeFormat, v); err != nil {
+ return time.Time{}, errors.New("cannot parse string time")
+ }
+ case int64:
+ calcTime = time.Unix(v, 0).UTC()
+ default:
+ return time.Time{}, errors.New("arg must be a timestamp")
+ }
+ return calcTime, nil
+}
+
+// minMaxViews returns the min and max view from a list of views
+// with a time quantum taken into consideration. It assumes that
+// all views represent the same base view name (the logic depends
+// on the views sorting correctly in alphabetical order).
+func minMaxViews(views []string, q TimeQuantum) (min string, max string) {
+ // Sort the list of views.
+ sort.Strings(views)
+
+ // Determine the least significant quantum and set that as the
+ // number of string characters to compare against.
+ var chars int
+ if q.HasYear() {
+ chars = 4
+ } else if q.HasMonth() {
+ chars = 6
+ } else if q.HasDay() {
+ chars = 8
+ } else if q.HasHour() {
+ chars = 10
+ }
+
+ // min: get the first view with the matching number of time chars.
+ for _, v := range views {
+ if len(viewTimePart(v)) == chars {
+ min = v
+ break
+ }
+ }
+
+ // max: get the first view (from the end) with the matching number of time chars.
+ for i := len(views) - 1; i >= 0; i-- {
+ if len(viewTimePart(views[i])) == chars {
+ max = views[i]
+ break
+ }
+ }
+
+ return min, max
+}
+
+// timeOfView returns a valid time.Time based on the view string.
+// For upper bound use, the result can be adjusted by one by setting
+// the `adj` argument to `true`.
+func timeOfView(v string, adj bool) (time.Time, error) {
+ if v == "" {
+ return time.Time{}, nil
+ }
+
+ layout := "2006010203"
+ timePart := viewTimePart(v)
+
+ switch len(timePart) {
+ case 4: // year
+ t, err := time.Parse(layout[:4], timePart)
+ if err != nil {
+ return time.Time{}, err
+ }
+ if adj {
+ t = t.AddDate(1, 0, 0)
+ }
+ return t, nil
+ case 6: // month
+ t, err := time.Parse(layout[:6], timePart)
+ if err != nil {
+ return time.Time{}, err
+ }
+ if adj {
+ t = addMonth(t)
+ }
+ return t, nil
+ case 8: // day
+ t, err := time.Parse(layout[:8], timePart)
+ if err != nil {
+ return time.Time{}, err
+ }
+ if adj {
+ t = t.AddDate(0, 0, 1)
+ }
+ return t, nil
+ case 10: // hour
+ t, err := time.Parse(layout[:10], timePart)
+ if err != nil {
+ return time.Time{}, err
+ }
+ if adj {
+ t = t.Add(time.Hour)
+ }
+ return t, nil
+ }
+
+ return time.Time{}, fmt.Errorf("invalid time format on view: %s", v)
+}
+
+// viewTimePart returns the time portion of a string view name.
+// e.g. the view "string_201901" would return "201901".
+func viewTimePart(v string) string {
+ parts := strings.Split(v, "_")
+ return parts[len(parts)-1]
+}
diff --git a/time_internal_test.go b/time_internal_test.go
index 18d5933be..58079380b 100644
--- a/time_internal_test.go
+++ b/time_internal_test.go
@@ -165,6 +165,131 @@ func TestViewsByTimeRange(t *testing.T) {
})
}
+func TestMinMaxViews(t *testing.T) {
+ t.Run("Combos", func(t *testing.T) {
+ tests := []struct {
+ views []string
+ q TimeQuantum
+ min string
+ max string
+ }{
+ {
+ []string{""},
+ mustParseTimeQuantum("Y"),
+ "",
+ "",
+ },
+ {
+ []string{"std_2019", "std_2020", "std_202002", "std_202002", "std_2022"},
+ mustParseTimeQuantum("Y"),
+ "std_2019",
+ "std_2022",
+ },
+ {
+ []string{"std_201902", "std_201901"},
+ mustParseTimeQuantum("M"),
+ "std_201901",
+ "std_201902",
+ },
+ {
+ []string{"std_201902", "std_201901"},
+ mustParseTimeQuantum("D"),
+ "",
+ "",
+ },
+ {
+ []string{"std_20190201"},
+ mustParseTimeQuantum("D"),
+ "std_20190201",
+ "std_20190201",
+ },
+ {
+ []string{"foo", "bar"},
+ mustParseTimeQuantum("D"),
+ "",
+ "",
+ },
+ }
+ for i, test := range tests {
+ if min, max := minMaxViews(test.views, test.q); min != test.min {
+ t.Errorf("test %d expected min: %v, but got: %v", i, test.min, min)
+ } else if max != test.max {
+ t.Errorf("test %d expected max: %v, but got: %v", i, test.max, max)
+ }
+ }
+ })
+}
+
+func TestTimeOfView(t *testing.T) {
+ t.Run("Combos", func(t *testing.T) {
+ tests := []struct {
+ view string
+ exp time.Time
+ expAdj time.Time
+ expErr string
+ }{
+ {
+ "std_2019",
+ time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC),
+ time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC),
+ "",
+ },
+ {
+ "std_201902",
+ time.Date(2019, 2, 1, 0, 0, 0, 0, time.UTC),
+ time.Date(2019, 3, 1, 0, 0, 0, 0, time.UTC),
+ "",
+ },
+ {
+ "std_20190203",
+ time.Date(2019, 2, 3, 0, 0, 0, 0, time.UTC),
+ time.Date(2019, 2, 4, 0, 0, 0, 0, time.UTC),
+ "",
+ },
+ {
+ "std_2019020308",
+ time.Date(2019, 2, 3, 8, 0, 0, 0, time.UTC),
+ time.Date(2019, 2, 3, 9, 0, 0, 0, time.UTC),
+ "",
+ },
+ {
+ "foo",
+ time.Time{},
+ time.Time{},
+ "invalid time format on view: foo",
+ },
+ {
+ "std_201902030801",
+ time.Time{},
+ time.Time{},
+ "invalid time format on view: std_201902030801",
+ },
+ }
+ for i, test := range tests {
+ // adj: false
+ if tm, err := timeOfView(test.view, false); err != nil {
+ if err.Error() != test.expErr {
+ t.Errorf("test %d got unexpected error: %s", i, err)
+ }
+ } else if test.expErr != "" {
+ t.Errorf("test %d expected error: %s but got none", i, test.expErr)
+ } else if tm != test.exp {
+ t.Errorf("test %d expected time: %v, but got: %v", i, test.exp, tm)
+ }
+ // adj: true
+ if tm, err := timeOfView(test.view, true); err != nil {
+ if err.Error() != test.expErr {
+ t.Errorf("test %d got unexpected error: %s", i, err)
+ }
+ } else if test.expErr != "" {
+ t.Errorf("test %d expected error: %s but got none", i, test.expErr)
+ } else if tm != test.expAdj {
+ t.Errorf("test %d expected time: %v, but got: %v", i, test.expAdj, tm)
+ }
+ }
+ })
+}
+
// defaultTimeLayout is the time layout used by the tests.
const defaultTimeLayout = "2006-01-02 15:04"