Merge pull request #1851 from travisturner/rows-time-range

add from/to range arguments to Rows() call

docs/query-language.md

@@ -812,10 +812,12 @@ Options(Row(f1=10), shards=[0, 2])
 {"attrs":{},"columns":[100, 2097152]}
 ```
 
+#### Rows
+
 **Spec:**
 
 ```
-Rows(<FIELD>, previous=<UINT|STRING>, limit=<UINT>, column=<UINT|STRING>)
+Rows(<FIELD>, previous=<UINT|STRING>, limit=<UINT>, column=<UINT|STRING>, from=<TIMESTAMP>, to=<TIMESTAMP>)
 ```
 
 **Description:**
@@ -832,6 +834,9 @@ is given, the number of rowIDs returned will be less than or equal to
 result sets. Results are always ordered, so setting `previous` as the last
 result of the previous request will start from the next available row.
 
+If the field is of type `time`, the `from` and `to` arguments can be provided
+to restrict the result to a specific time span. If `from` and `to` are
+not provided, the full range of existing data will be queried.
 
 **Result Type:** Object with `"rows" or "keys" and an array of integers or strings respectively.`
 

executor.go

@@ -1152,16 +1152,75 @@ func (e *executor) executeRowsShard(_ context.Context, index string, fieldName s
 		return nil, ErrFieldNotFound
 	}
 
-	// Rows query does not currently support a `time` field that has
-	// `noStandardView: true`.
-	// TODO https://github.com/pilosa/pilosa/issues/1783
-	if f.Type() == FieldTypeTime && f.options.NoStandardView {
-		return nil, errors.New("Rows() query on time field with no standard view is not currently supported")
-	}
+	// rowIDs is the result set.
+	var rowIDs RowIDs
 
-	frag := e.Holder.fragment(index, fieldName, viewStandard, shard)
-	if frag == nil {
-		return make(RowIDs, 0), nil
+	// views contains the list of views to inspect (and merge)
+	// in order to represent `Rows` for the field.
+	var views []string = []string{viewStandard}
+
+	// Handle `time` fields.
+	if f.Type() == FieldTypeTime {
+		var err error
+
+		// Parse "from" time, if set.
+		var fromTime time.Time
+		if v, ok := c.Args["from"]; ok {
+			if fromTime, err = parseTime(v); err != nil {
+				return nil, errors.Wrap(err, "parsing from time")
+			}
+		}
+
+		// Parse "to" time, if set.
+		var toTime time.Time
+		if v, ok := c.Args["to"]; ok {
+			if toTime, err = parseTime(v); err != nil {
+				return nil, errors.Wrap(err, "parsing to time")
+			}
+		}
+
+		// Calculate the views for a range as long as some piece of the range
+		// (from/to) are specified, or if there's no standard view to represent
+		// all dates.
+		if !fromTime.IsZero() || !toTime.IsZero() || f.options.NoStandardView {
+			// If no quantum exists then return an empty result set.
+			q := f.TimeQuantum()
+			if q == "" {
+				return rowIDs, nil
+			}
+
+			// Get min/max based on existing views.
+			var vs []string
+			for _, v := range f.views() {
+				vs = append(vs, v.name)
+			}
+			min, max := minMaxViews(vs, q)
+
+			// If min/max are empty, there were no time views.
+			if min == "" || max == "" {
+				return rowIDs, nil
+			}
+
+			// Convert min/max from string to time.Time.
+			minTime, err := timeOfView(min, false)
+			if err != nil {
+				return rowIDs, errors.Wrapf(err, "getting min time from view: %s", min)
+			}
+			if fromTime.IsZero() || fromTime.Before(minTime) {
+				fromTime = minTime
+			}
+
+			maxTime, err := timeOfView(max, true)
+			if err != nil {
+				return rowIDs, errors.Wrapf(err, "getting max time from view: %s", max)
+			}
+			if toTime.IsZero() || toTime.After(maxTime) {
+				toTime = maxTime
+			}
+
+			// Determine the views based on the specified time range.
+			views = viewsByTimeRange(viewStandard, fromTime, toTime, q)
+		}
 	}
 
 	start := uint64(0)
@@ -1177,17 +1236,30 @@ func (e *executor) executeRowsShard(_ context.Context, index string, fieldName s
 	} else if ok {
 		colShard := columnID >> shardWidthExponent
 		if colShard != shard {
-			return RowIDs{}, nil
+			return rowIDs, nil
 		}
 		filters = append(filters, filterColumn(columnID))
 	}
-	if limit, hasLimit, err := c.UintArg("limit"); err != nil {
+
+	limit := int(^uint(0) >> 1)
+	if lim, hasLimit, err := c.UintArg("limit"); err != nil {
 		return nil, errors.Wrap(err, "getting limit")
 	} else if hasLimit {
-		filters = append(filters, filterWithLimit(limit))
+		filters = append(filters, filterWithLimit(lim))
+		limit = int(lim)
 	}
 
-	return frag.rows(start, filters...), nil
+	for _, view := range views {
+		frag := e.Holder.fragment(index, fieldName, view, shard)
+		if frag == nil {
+			continue
+		}
+
+		viewRows := frag.rows(start, filters...)
+		rowIDs = rowIDs.merge(viewRows, limit)
+	}
+
+	return rowIDs, nil
 }
 
 func (e *executor) executeRowShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
@@ -1228,31 +1300,17 @@ func (e *executor) executeRowShard(ctx context.Context, index string, c *pql.Cal
 
 	// Parse "from" time, if set.
 	var fromTime time.Time
-	if _, ok := c.Args["from"]; ok {
-		switch v := c.Args["from"].(type) {
-		case string:
-			if fromTime, err = time.Parse(TimeFormat, v); err != nil {
-				return nil, errors.New("cannot parse Row() 'from' time")
-			}
-		case int64:
-			fromTime = time.Unix(v, 0).UTC()
-		default:
-			return nil, errors.New("Row() 'from' arg must be a timestamp")
+	if v, ok := c.Args["from"]; ok {
+		if fromTime, err = parseTime(v); err != nil {
+			return nil, errors.Wrap(err, "parsing from time")
 		}
 	}
 
 	// Parse "to" time, if set.
 	var toTime time.Time
-	if _, ok := c.Args["to"]; ok {
-		switch v := c.Args["to"].(type) {
-		case string:
-			if toTime, err = time.Parse(TimeFormat, v); err != nil {
-				return nil, errors.New("cannot parse Row() 'to' time")
-			}
-		case int64:
-			toTime = time.Unix(v, 0).UTC()
-		default:
-			return nil, errors.New("Row() 'to' arg must be a timestamp")
+	if v, ok := c.Args["to"]; ok {
+		if toTime, err = parseTime(v); err != nil {
+			return nil, errors.Wrap(err, "parsing to time")
 		}
 	}
 

executor_test.go

@@ -3131,13 +3131,51 @@ func TestExecutor_Execute_Rows(t *testing.T) {
 }
 
 func TestExecutor_Execute_RowsTime(t *testing.T) {
+	writeQuery := fmt.Sprintf(`
+		Set(9, f=1, 2001-01-01T00:00)
+		Set(9, f=2, 2002-01-01T00:00)
+		Set(9, f=3, 2003-01-01T00:00)
+		Set(9, f=4, 2004-01-01T00:00)
+
+		Set(%d, f=13, 2003-02-02T00:00)
+		`, pilosa.ShardWidth+9)
+	readQueries := []string{
+		`Rows(f, from=1999-12-31T00:00, to=2002-01-01T03:00)`,
+		`Rows(f, from=2002-01-01T00:00, to=2004-01-01T00:00)`,
+		`Rows(f, from=1990-01-01T00:00, to=1999-01-01T00:00)`,
+		`Rows(f)`,
+		`Rows(f, from=2002-01-01T00:00)`,
+		`Rows(f, to=2003-02-03T00:00)`,
+	}
+	expResults := [][]uint64{
+		{1},
+		{2, 3, 13},
+		{},
+		{1, 2, 3, 4, 13},
+		{2, 3, 4, 13},
+		{1, 2, 3, 13},
+	}
+
+	responses := runCallTest(t, writeQuery, readQueries,
+		nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), true))
+
+	for i := range responses {
+		t.Run(fmt.Sprintf("response-%d", i), func(t *testing.T) {
+			if rows := responses[i].Results[0].(pilosa.RowIdentifiers).Rows; !reflect.DeepEqual(rows, expResults[i]) {
+				t.Fatalf("unexpected rows: %+v", rows)
+			}
+		})
+	}
+}
+
+// Ensure that an empty time field returns empty Rows().
+func TestExecutor_Execute_RowsTimeEmpty(t *testing.T) {
 	c := test.MustRunCluster(t, 1)
 	defer c.Close()
-	c.CreateField(t, "i", pilosa.IndexOptions{}, "t", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), true))
-
-	exp := "executing: Rows() query on time field with no standard view is not currently supported"
-	if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Rows(field=t)`}); err == nil || err.Error() != exp {
-		t.Fatalf("expected error: %s", exp)
+	c.CreateField(t, "i", pilosa.IndexOptions{}, "x", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), true))
+	rows := c.Query(t, "i", `Rows(x, from=1999-12-31T00:00, to=2002-01-01T03:00)`).Results[0].(pilosa.RowIdentifiers).Rows
+	if !reflect.DeepEqual(rows, []uint64{}) {
+		t.Fatalf("unexpected rows: %+v", rows)
 	}
 }
 

time.go

@@ -17,6 +17,7 @@ package pilosa
 import (
 	"errors"
 	"fmt"
+	"sort"
 	"strings"
 	"time"
 )
@@ -214,3 +215,120 @@ func nextDayGTE(t time.Time, end time.Time) bool {
 	}
 	return end.After(next)
 }
+
+// parseTime parses a string or int64 into a time.Time value.
+func parseTime(t interface{}) (time.Time, error) {
+	var err error
+	var calcTime time.Time
+	switch v := t.(type) {
+	case string:
+		if calcTime, err = time.Parse(TimeFormat, v); err != nil {
+			return time.Time{}, errors.New("cannot parse string time")
+		}
+	case int64:
+		calcTime = time.Unix(v, 0).UTC()
+	default:
+		return time.Time{}, errors.New("arg must be a timestamp")
+	}
+	return calcTime, nil
+}
+
+// minMaxViews returns the min and max view from a list of views
+// with a time quantum taken into consideration. It assumes that
+// all views represent the same base view name (the logic depends
+// on the views sorting correctly in alphabetical order).
+func minMaxViews(views []string, q TimeQuantum) (min string, max string) {
+	// Sort the list of views.
+	sort.Strings(views)
+
+	// Determine the least significant quantum and set that as the
+	// number of string characters to compare against.
+	var chars int
+	if q.HasYear() {
+		chars = 4
+	} else if q.HasMonth() {
+		chars = 6
+	} else if q.HasDay() {
+		chars = 8
+	} else if q.HasHour() {
+		chars = 10
+	}
+
+	// min: get the first view with the matching number of time chars.
+	for _, v := range views {
+		if len(viewTimePart(v)) == chars {
+			min = v
+			break
+		}
+	}
+
+	// max: get the first view (from the end) with the matching number of time chars.
+	for i := len(views) - 1; i >= 0; i-- {
+		if len(viewTimePart(views[i])) == chars {
+			max = views[i]
+			break
+		}
+	}
+
+	return min, max
+}
+
+// timeOfView returns a valid time.Time based on the view string.
+// For upper bound use, the result can be adjusted by one by setting
+// the `adj` argument to `true`.
+func timeOfView(v string, adj bool) (time.Time, error) {
+	if v == "" {
+		return time.Time{}, nil
+	}
+
+	layout := "2006010203"
+	timePart := viewTimePart(v)
+
+	switch len(timePart) {
+	case 4: // year
+		t, err := time.Parse(layout[:4], timePart)
+		if err != nil {
+			return time.Time{}, err
+		}
+		if adj {
+			t = t.AddDate(1, 0, 0)
+		}
+		return t, nil
+	case 6: // month
+		t, err := time.Parse(layout[:6], timePart)
+		if err != nil {
+			return time.Time{}, err
+		}
+		if adj {
+			t = addMonth(t)
+		}
+		return t, nil
+	case 8: // day
+		t, err := time.Parse(layout[:8], timePart)
+		if err != nil {
+			return time.Time{}, err
+		}
+		if adj {
+			t = t.AddDate(0, 0, 1)
+		}
+		return t, nil
+	case 10: // hour
+		t, err := time.Parse(layout[:10], timePart)
+		if err != nil {
+			return time.Time{}, err
+		}
+		if adj {
+			t = t.Add(time.Hour)
+		}
+		return t, nil
+	}
+
+	return time.Time{}, fmt.Errorf("invalid time format on view: %s", v)
+}
+
+// viewTimePart returns the time portion of a string view name.
+// e.g. the view "string_201901" would return "201901".
+func viewTimePart(v string) string {
+	parts := strings.Split(v, "_")
+	return parts[len(parts)-1]
+}

time_internal_test.go

@@ -165,6 +165,131 @@ func TestViewsByTimeRange(t *testing.T) {
 	})
 }
 
+func TestMinMaxViews(t *testing.T) {
+	t.Run("Combos", func(t *testing.T) {
+		tests := []struct {
+			views []string
+			q     TimeQuantum
+			min   string
+			max   string
+		}{
+			{
+				[]string{""},
+				mustParseTimeQuantum("Y"),
+				"",
+				"",
+			},
+			{
+				[]string{"std_2019", "std_2020", "std_202002", "std_202002", "std_2022"},
+				mustParseTimeQuantum("Y"),
+				"std_2019",
+				"std_2022",
+			},
+			{
+				[]string{"std_201902", "std_201901"},
+				mustParseTimeQuantum("M"),
+				"std_201901",
+				"std_201902",
+			},
+			{
+				[]string{"std_201902", "std_201901"},
+				mustParseTimeQuantum("D"),
+				"",
+				"",
+			},
+			{
+				[]string{"std_20190201"},
+				mustParseTimeQuantum("D"),
+				"std_20190201",
+				"std_20190201",
+			},
+			{
+				[]string{"foo", "bar"},
+				mustParseTimeQuantum("D"),
+				"",
+				"",
+			},
+		}
+		for i, test := range tests {
+			if min, max := minMaxViews(test.views, test.q); min != test.min {
+				t.Errorf("test %d expected min: %v, but got: %v", i, test.min, min)
+			} else if max != test.max {
+				t.Errorf("test %d expected max: %v, but got: %v", i, test.max, max)
+			}
+		}
+	})
+}
+
+func TestTimeOfView(t *testing.T) {
+	t.Run("Combos", func(t *testing.T) {
+		tests := []struct {
+			view   string
+			exp    time.Time
+			expAdj time.Time
+			expErr string
+		}{
+			{
+				"std_2019",
+				time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC),
+				time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC),
+				"",
+			},
+			{
+				"std_201902",
+				time.Date(2019, 2, 1, 0, 0, 0, 0, time.UTC),
+				time.Date(2019, 3, 1, 0, 0, 0, 0, time.UTC),
+				"",
+			},
+			{
+				"std_20190203",
+				time.Date(2019, 2, 3, 0, 0, 0, 0, time.UTC),
+				time.Date(2019, 2, 4, 0, 0, 0, 0, time.UTC),
+				"",
+			},
+			{
+				"std_2019020308",
+				time.Date(2019, 2, 3, 8, 0, 0, 0, time.UTC),
+				time.Date(2019, 2, 3, 9, 0, 0, 0, time.UTC),
+				"",
+			},
+			{
+				"foo",
+				time.Time{},
+				time.Time{},
+				"invalid time format on view: foo",
+			},
+			{
+				"std_201902030801",
+				time.Time{},
+				time.Time{},
+				"invalid time format on view: std_201902030801",
+			},
+		}
+		for i, test := range tests {
+			// adj: false
+			if tm, err := timeOfView(test.view, false); err != nil {
+				if err.Error() != test.expErr {
+					t.Errorf("test %d got unexpected error: %s", i, err)
+				}
+			} else if test.expErr != "" {
+				t.Errorf("test %d expected error: %s but got none", i, test.expErr)
+			} else if tm != test.exp {
+				t.Errorf("test %d expected time: %v, but got: %v", i, test.exp, tm)
+			}
+			// adj: true
+			if tm, err := timeOfView(test.view, true); err != nil {
+				if err.Error() != test.expErr {
+					t.Errorf("test %d got unexpected error: %s", i, err)
+				}
+			} else if test.expErr != "" {
+				t.Errorf("test %d expected error: %s but got none", i, test.expErr)
+			} else if tm != test.expAdj {
+				t.Errorf("test %d expected time: %v, but got: %v", i, test.expAdj, tm)
+			}
+		}
+	})
+}
+
 // defaultTimeLayout is the time layout used by the tests.
 const defaultTimeLayout = "2006-01-02 15:04"